def run(mpi_comm,
state='SA',
state_sub='SA',
metadata_dir='00_data_virtual_link/03_metadata_dir/SA',
data_location='00_data_virtual_link/03_data_location_SA',
root_dir='03_workspace',
log_prefnm='03_workspace/SA_mpi_log'):
# this is the timezone we are working in
timezone = "Australia/Adelaide"
### create list of metadata files to add to netCDF files
metadata_dir = metadata_dir
exclude_folders = [
'WA11', 'WA12', 'WA28', 'WASA352', 'auslamp_metadata_template.txt'
] ### exclude these for now as they are different instruments
metadata_files = []
for file in os.listdir(metadata_dir):
if file.endswith('.txt'):
if not file.startswith(("WA11", "WA12", "WA28", "WASA352",
"auslamp_metadata_template.txt")):
metadata_files.append(file)
metadata_files = sorted(metadata_files)
metadata_template = 'auslamp_metadata_template.txt' ### ASEG sample metadata template
metadata_template_loc = path.join(metadata_dir, metadata_template)
channels = ['BY', 'BX', 'EX', 'EY', 'BZ', 'TP', 'ambientTemperature']
channels_b = ['BY', 'BX', 'EX', 'EY', 'BZ']
state = state
state_sub = state_sub
data_type = 'Level_0_Concatinated_Time_Series_NetCDF'
period = 'Long_period' # BB or Long_Period
survey_name = 'SA_AusLAMP_MT_Survey_Musgraves_APY_2016_to_2018'
frequency = 10
data_location = data_location
root_dir = root_dir
out_location = path.join(root_dir, state, period, survey_name, state_sub,
data_type)
###
#
###
rank = mpi_comm.Get_rank()
if rank == 0:
if not os.path.exists(out_location):
os.makedirs(out_location)
mpi_comm.Barrier()
###
exclude = ['Plots', 'config', 'log', 'temp', 'old']
exclude_folders = ['WA11', 'WA12', 'WA28', 'WASA352']
metadata_dirs = ['config']
search = path.join(data_location, '*')
subdirs = sorted(
[i for i in glob(search) if i.split('/')[-1] not in exclude_folders])
###
#
###
rank = mpi_comm.Get_rank()
size = mpi_comm.Get_size()
chunk = int(len(subdirs) / size) + 1
i1 = rank * chunk
i2 = i1 + chunk
logfnm = log_prefnm + '_%03d.txt' % (rank)
fp = open(logfnm, 'w')
print('>>> all subdirs: \n\t%s\n\n' % '\n\t'.join(subdirs),
file=fp,
flush=True)
print('>>> work[%d,%d): \n\t%s\n\n' %
(i1, i2, '\n\t'.join(subdirs[i1:i2])),
file=fp,
flush=True)
###
#
###
for idx, subdir in enumerate(subdirs[i1:i2]):
if subdir.split('/')[-1].upper() not in exclude_folders:
print('>>> (%d/%d) %s' % (idx, i2 - i1, subdir),
file=fp,
flush=True)
syte_name = subdir.split('/')[-1].upper()
output_dir = path.join(out_location, syte_name.upper())
if not os.path.exists(output_dir):
os.makedirs(output_dir)
search2 = path.join(subdir, '*')
data_dirs = [sorted([i for i in glob(search2)])]
for folder in data_dirs:
day_name = [eye.split('/')[-1] for eye in folder]
for fold, day in zip(folder, day_name):
out_dir = path.join(output_dir, day)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
channel_files_ex = sorted(
[i for i in os.listdir(fold) if i.endswith(("EX"))])
channel_files_ey = sorted(
[i for i in os.listdir(fold) if i.endswith(("EY"))])
channel_files_bx = sorted(
[i for i in os.listdir(fold) if i.endswith(("BX"))])
channel_files_by = sorted(
[i for i in os.listdir(fold) if i.endswith(("BY"))])
channel_files_bz = sorted(
[i for i in os.listdir(fold) if i.endswith(("BZ"))])
names = [channel_files_ex[0].split('.')[0] + '.nc']
for name in names:
out_name = path.join(out_dir, name)
for i, j, k, l, m in zip(channel_files_ex,
channel_files_ey,
channel_files_bx,
channel_files_by,
channel_files_bz):
for metadata in metadata_files:
if metadata.split('.')[0] == syte_name:
metadata_loc = path.join(
metadata_dir, metadata)
fd = open(metadata_loc, 'r')
lines = [line for line in fd]
lines = [line.rstrip('\n') for line in lines]
fd2 = open(metadata_template_loc, 'r')
auslamp_metadata = [line for line in fd2]
auslamp_metadata = [
line.rstrip('\n')
for line in auslamp_metadata
]
auslamp_metadata_no_equal = [
s.strip(' =') for s in auslamp_metadata
]
lines_and_metadata = []
for f, b in zip(auslamp_metadata_no_equal,
lines):
if b != "NA":
lines_and_metadata.append([f, b])
for f, b in zip(
[i[0] for i in lines_and_metadata],
[i[1] for i in lines_and_metadata]):
globals()[f] = b
EX1 = []
EY1 = []
BX1 = []
BY1 = []
BZ1 = []
df_bx = open(path.join(fold, k))
for line in df_bx:
BX1.append(line)
df_by = open(path.join(fold, l))
for line in df_by:
BY1.append(line)
df_bz = open(path.join(fold, m))
for line in df_bz:
BZ1.append(line)
df_ex = open(path.join(fold, i))
for line in df_ex:
EX1.append(line)
df_ey = open(path.join(fold, j))
for line in df_ey:
EY1.append(line)
timestamp = i.split('_')[1].split('.')[0]
unixtime = timestamp_to_timeaxis(timestamp, timezone)
t1 = float(unixtime)
number_of_samples = len(EX1)
seconds = number_of_samples / frequency
t2 = t1 + seconds
time1 = np.linspace(t1, t2, num=len(EX1))
dataset_P = Dataset(out_name, 'w', format='NETCDF4')
bx = dataset_P.createDimension('bx', len(BX1))
by = dataset_P.createDimension('by', len(BY1))
bz = dataset_P.createDimension('bz', len(BZ1))
ex = dataset_P.createDimension('ex', len(EX1))
ey = dataset_P.createDimension('ey', len(EY1))
time = dataset_P.createDimension('time', len(time1))
ex = dataset_P.createVariable('ex', np.float32,
('ex', ))
ex.units = 'mV'
ex.long_name = 'electric field in N-S orientation'
ex.standard_name = 'electric field'
try:
ex.sampling_rate_Hz = Ex_NS_sample_rate_samples_per_second
except Exception:
pass
try:
ex.NS_electrode_dipole_length_m = Ex_NS_electrode_dipole_length_m
except Exception:
pass
try:
ex.NS_electrode_azimuth_dec_deg = Ex_NS_electrode_azimuth_dec_deg
except Exception:
pass
try:
ex.NS_field_gain = Ex_NS_field_gain
except Exception:
pass
try:
ex.North_length_m = Ex_North_length_m
except Exception:
pass
try:
ex.North_electrode_number = Ex_North_electrode_number
except Exception:
pass
try:
ex.North_resistance_to_ground_kOhms = Ex_North_resistance_to_ground_kOhms
except Exception:
pass
try:
ex.South_length_m = Ex_South_length_m
except Exception:
pass
try:
ex.South_electrode_number = Ex_South_electrode_number
except Exception:
pass
try:
ex.South_resistance_to_ground_kOhms = Ex_South_resistance_to_ground_kOhms
except Exception:
pass
try:
ex.ground_center_electrode_number = ground_center_electrode_number
except Exception:
pass
try:
ex.N_S_center_resistance_to_ground_kOhms = Ex_N_S_center_resistance_to_ground_kOhms
except Exception:
pass
try:
ex.NS_AC_mV = Ex_NS_AC_mV
except Exception:
pass
try:
ex.NS_DC_mV = Ex_NS_DC_mV
except Exception:
pass
try:
ex.MT_recorder_Ex_NS_voltage_mV = MT_recorder_Ex_NS_voltage_mV
except Exception:
pass
ex[:] = EX1
ey = dataset_P.createVariable('ey', np.float32,
('ey', ))
ey.units = 'mV'
ey.long_name = 'electric field in E-W orientation'
ey.standard_name = 'electric field'
try:
ey.sampling_rate_Hz = Ey_EW_sample_rate_samples_per_second
except Exception:
pass
try:
ey.EW_electrode_dipole_length_m = Ey_EW_electrode_dipole_length_m
except Exception:
pass
try:
ey.EW_electrode_azimuth_dec_deg = Ey_EW_electrode_azimuth_dec_deg
except Exception:
pass
try:
ey.EW_field_gain = Ey_EW_field_gain
except Exception:
pass
try:
ey.East_length_m = Ey_East_length_m
except Exception:
pass
try:
ey.East_electrode_number = Ey_East_electrode_number
except Exception:
pass
try:
ey.East_resistance_to_ground_kOhms = Ey_East_resistance_to_ground_kOhms
except Exception:
pass
try:
ey.West_length_m = Ey_West_length_m
except Exception:
pass
try:
ey.West_electrode_number = Ey_West_electrode_number
except Exception:
pass
try:
ey.West_resistance_to_ground_kOhms = Ey_West_resistance_to_ground_kOhms
except Exception:
pass
try:
ey.ground_center_electrode_number = ground_center_electrode_number
except Exception:
pass
try:
ey.E_W_center_resistance_to_ground_kOhms = Ey_E_W_center_resistance_to_ground_kOhms
except Exception:
pass
try:
ey.EW_AC_mV = Ey_EW_AC_mV
except Exception:
pass
try:
ey.EW_DC_mV = Ey_EW_DC_mV
except Exception:
pass
try:
ey.MT_recorder_Ey_EW_voltage_mV = MT_recorder_Ey_EW_voltage_mV
except Exception:
pass
ey[:] = EY1
bx = dataset_P.createVariable('bx', np.float32,
('bx', ))
bx.units = 'microVolts'
bx.long_name = 'instrument recorded magnetic field in X direction (microVolts)'
bx.standard_name = 'magnetic field'
try:
bx.sampling_rate_Hz = Bx_sampling_rate
except Exception:
pass
try:
bx.azimuth_degrees = Bx_azimuth_degrees
except Exception:
pass
try:
bx.coil_number = Bx_coil_number
except Exception:
pass
bx[:] = BX1
by = dataset_P.createVariable('by', np.float32,
('by', ))
by.units = 'microVolts'
by.long_name = 'instrument recorded magnetic field in Y direction (microVolts)'
by.standard_name = 'magnetic field'
try:
by.sampling_rate_Hz = By_sampling_rate
except Exception:
pass
try:
by.azimuth_degrees = By_azimuth_degrees
except Exception:
pass
try:
by.coil_number = By_coil_number
except Exception:
pass
by[:] = BY1
bz = dataset_P.createVariable('bz', np.float32,
('bz', ))
bz.units = 'microVolts'
bz.long_name = 'instrument recorded magnetic field in Z direction (microVolts)'
bz.standard_name = 'magnetic field'
try:
bz.sampling_rate_Hz = Bz_sampling_rate
except Exception:
pass
try:
bz.azimuth_degrees = Bz_azimuth_degrees
except Exception:
pass
try:
bz.coil_number = Bz_coil_number
except Exception:
pass
bz[:] = BZ1
time = dataset_P.createVariable(
'time', np.float32, ('time', ))
time.units = 'seconds'
time.long_name = 'time'
time.standard_name = 'time'
try:
time.sampling_rate_Hz = Bz_sampling_rate
except Exception:
pass
time[:] = time1
try:
dataset_P.title = deployment_survey_prospect_name
except Exception:
pass
try:
dataset_P.summary = 'Long period time series for site' + ' ' + site_number + ' ' + 'of the' + ' ' + deployment_survey_prospect_name + ' ' + 'survey'
except Exception:
pass
try:
dataset_P.source = operator
except Exception:
pass
try:
dataset_P.date_created = current_time
except Exception:
pass
try:
dataset_P.Conventions = 'ACDD-1.3'
except Exception:
pass
#dataset_P.metadata_link = ' https://geoenetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/fxxxxx'
try:
dataset_P.license = 'Creative Commons Attribution 4.0 International (CC BY 4.0)'
except Exception:
pass
### Deployment metadata
try:
dataset_P.site_number = site_number
except Exception:
pass
try:
dataset_P.site_name = site_name
except Exception:
pass
try:
dataset_P.location = location
except Exception:
pass
try:
dataset_P.deployment_survey_prospect_name = deployment_survey_prospect_name
except Exception:
pass
try:
dataset_P.institution_agency = institution_agency
except Exception:
pass
try:
dataset_P.AUSGRID_number = AUSGRID_number
except Exception:
pass
try:
dataset_P.operator = operator
except Exception:
pass
try:
dataset_P.recorder_latitude_dec_deg = recorder_latitude_dec_deg
except Exception:
pass
try:
dataset_P.recorder_longitude_dec_deg = recorder_longitude_dec_deg
except Exception:
pass
try:
dataset_P.recorder_GPS_projection_system = recorder_GPS_projection_system
except Exception:
pass
try:
dataset_P.field_GPS_latitude_dec_deg = field_GPS_latitude_dec_deg
except Exception:
pass
try:
dataset_P.field_GPS_longitude_dec_deg = field_GPS_longitude_dec_deg
except Exception:
pass
try:
dataset_P.GPS_easting_AMG = GPS_easting_AMG
except Exception:
pass
try:
dataset_P.GPS_northing_AMG = GPS_northing_AMG
except Exception:
pass
try:
dataset_P.AMP_zone = AMG_zone
except Exception:
pass
try:
dataset_P.GPS_projection_system = GPS_projection_system
except Exception:
pass
try:
dataset_P.altitude_or_elevation_from_GPS_or_instrument = altitude_or_elevation_from_GPS_or_instrument
except Exception:
pass
try:
dataset_P.deployment_date = deployment_date
except Exception:
pass
try:
dataset_P.LOCAL_start_hour = LOCAL_start_hour
except Exception:
pass
try:
dataset_P.LOCAL_start_minute = LOCAL_start_minute
except Exception:
pass
try:
dataset_P.LOCAL_start_second = LOCAL_start_second
except Exception:
pass
try:
dataset_P.UTC_date_from_instrument = UTC_date_from_instrument
except Exception:
pass
try:
dataset_P.UTC_start_year = UTC_start_year
except Exception:
pass
try:
dataset_P.UTC_start_month = UTC_start_month
except Exception:
pass
try:
dataset_P.UTC_start_day = UTC_start_day
except Exception:
pass
try:
dataset_P.UTC_start_time = UTC_start_time
except Exception:
pass
try:
dataset_P.UTC_start_time_hour = UTC_start_time_hour
except Exception:
pass
try:
dataset_P.UTC_start_time_minute = UTC_start_time_minute
except Exception:
pass
try:
dataset_P.UTC_start_time_second = UTC_start_time_second
except Exception:
pass
try:
dataset_P.deployment_Julian_day = deployment_Julian_day
except Exception:
pass
try:
dataset_P.recording_method = recording_method
except Exception:
pass
try:
dataset_P.MT_recorder_type_model = MT_recorder_type_model
except Exception:
pass
try:
dataset_P.magnetometer_type_model = magnetometer_type_model
except Exception:
pass
try:
dataset_P.electrode_type_model = electrode_type_model
except Exception:
pass
try:
dataset_P.power_source_type_or_model = power_source_type_or_model
except Exception:
pass
try:
dataset_P.data_confidentiality = data_confidentiality
except Exception:
pass
try:
dataset_P.north_reference = north_reference
except Exception:
pass
try:
dataset_P.drift_calculation_end_time_GPS = drift_calculation_end_time_GPS
except Exception:
pass
try:
dataset_P.drift_calculation_end_time_instrument = drift_calculation_end_time_instrument
except Exception:
pass
try:
dataset_P.instrument_drift_in_seconds = instrument_drift_in_seconds
except Exception:
pass
try:
dataset_P.battery_voltage_no_load_V = battery_voltage_no_load_V
except Exception:
pass
try:
dataset_P.battery_voltage_on_load_V = battery_voltage_on_load_V
except Exception:
pass
try:
dataset_P.battery_voltage_MT_data_logger_V = battery_voltage_MT_data_logger_V
except Exception:
pass
try:
dataset_P.MT_box_case_number = MT_box_case_number
except Exception:
pass
try:
dataset_P.MT_recorder_data_logger_number = MT_recorder_data_logger_number
except Exception:
pass
try:
dataset_P.MT_recorder_interface_box_number = MT_recorder_interface_box_number
except Exception:
pass
try:
dataset_P.data_storage_device_number = data_storage_device_number
except Exception:
pass
try:
dataset_P.magnetometer_type = magnetometer_type
except Exception:
pass
try:
dataset_P.data_file_length_minutes_per_file = data_file_length_minutes_per_file
except Exception:
pass
try:
dataset_P.NS_EW_resistance_kOhms = NS_EW_resistance_kOhms
except Exception:
pass
try:
dataset_P.NS_EW_voltage_mV = NS_EW_voltage_mV
except Exception:
pass
try:
dataset_P.deployment_comments = deployment_comments
except Exception:
pass
try:
dataset_P.photo_taken_deployment = photo_taken_deployment
except Exception:
pass
try:
dataset_P.photo_identification_numbers_deployment = photo_identification_numbers_deployment
except Exception:
pass
#dataset_P.deployment_sheet_scan = deployment_sheet_scan
try:
dataset_P.calculated_date_for_retrieval = calculated_date_for_retrieval
except Exception:
pass
try:
dataset_P.retrieval_date_local = retrieval_date_local
except Exception:
pass
try:
dataset_P.retrieval_local_start_year = retrieval_local_start_year
except Exception:
pass
try:
dataset_P.retrieval_local_start_month = retrieval_local_start_month
except Exception:
pass
try:
dataset_P.retrieval_local_start_day = retrieval_local_start_day
except Exception:
pass
try:
dataset_P.retrieval_time_local = retrieval_time_local
except Exception:
pass
try:
dataset_P.retrieval_local_start_hour = retrieval_local_start_hour
except Exception:
pass
try:
dataset_P.retrieval_local_start_minute = retrieval_local_start_minute
except Exception:
pass
try:
dataset_P.retrieval_local_start_second = retrieval_local_start_second
except Exception:
pass
try:
dataset_P.retrieval_date_UTC = retrieval_date_UTC
except Exception:
pass
try:
dataset_P.retrieval_time_UTC = retrieval_time_UTC
except Exception:
pass
try:
dataset_P.retrieval_Julian_day = retrieval_Julian_day
except Exception:
pass
try:
dataset_P.retrieval_Ex_N_S_Center_resistance_kOhms = retrieval_Ex_N_S_Center_resistance_kOhms
except Exception:
pass
try:
dataset_P.retrieval_Ex_AC_mV = retrieval_Ex_AC_mV
except Exception:
pass
try:
dataset_P.retrieval_Ex_DC_mV = retrieval_Ex_DC_mV
except Exception:
pass
try:
dataset_P.retrieval_Ey_E_W_Center_resistance_kOhms = retrieval_Ey_E_W_Center_resistance_kOhms
except Exception:
pass
try:
dataset_P.retrieval_ey_AC_mV = retrieval_ey_AC_mV
except Exception:
pass
try:
dataset_P.retrieval_Ey_DC_mV = retrieval_Ey_DC_mV
except Exception:
pass
try:
dataset_P.retrieval_NS_EW_resistance_kOhms = retrieval_NS_EW_resistance_kOhms
except Exception:
pass
try:
dataset_P.MT_unit_operational = MT_unit_operational
except Exception:
pass
try:
dataset_P.retrieval_battery_voltage_V = retrieval_battery_voltage_V
except Exception:
pass
try:
dataset_P.electrode_cable_status = electrode_cable_status
except Exception:
pass
try:
dataset_P.mag_cable_status = mag_cable_status
except Exception:
pass
try:
dataset_P.retrieval_comments = retrieval_comments
except Exception:
pass
try:
dataset_P.days_operated = days_operated
except Exception:
pass
try:
dataset_P.photos_taken_retrieval = photos_taken_retrieval
except Exception:
pass
try:
dataset_P.photo_identification_numbers_retrieval = photo_identification_numbers_retrieval
except Exception:
pass
try:
dataset_P.site_data_quality_retrieval = site_data_quality_retrieval
except Exception:
pass
try:
dataset_P.redeployed = redeployed
except Exception:
pass
dataset_P.close()
mpi_comm.Barrier()
