def test_write_backup(self):
response_entry = {}
response_entry['n_i'] = 1
response_entry['bit_weight/value_d'] = 2
response_entry['gain/value_i'] = 1
response_entry['response_file_das_a'] = "das_path"
response_entry['response_file_sensor_a'] = "sens_path"
ref = columns.TABLES['/Experiment_g/Responses_g/Response_t']
columns.populate(ref, response_entry, None)
self.n_i_fix.ph5.read_response_t()
response_t = tabletokef.Rows_Keys(self.n_i_fix.ph5.Response_t['rows'],
self.n_i_fix.ph5.Response_t['keys'])
with LogCapture() as log:
resp_load.write_backup(response_t,
'/Experiment_g/Responses_g/Response_t',
'Response_t')
msg = log.records[0].msg
self.assertEqual(msg[0:33] + msg[40:],
'Writing table backup: Response_t__00.kef.')
filename = msg.split(': ')[1][:-1]
with open(os.path.join(self.tmpdir, filename), 'r') as content_file:
content = content_file.read().strip().split("Table row 1")[1]
self.assertEqual(
content, "\n/Experiment_g/Responses_g/Response_t"
"\n\tn_i=1"
"\n\tbit_weight/value_d=2.0"
"\n\tbit_weight/units_s="
"\n\tgain/units_s="
"\n\tgain/value_i=1"
"\n\tresponse_file_a="
"\n\tresponse_file_das_a=das_path"
"\n\tresponse_file_sensor_a=sens_path")
def update():
global EX, ARRAY_NAME, KEF
# There is a bug in batch update that kills kv
k = kef.Kef(KEF)
k.open()
k.read()
k.rewind()
ARRAY_NAME = None
while True:
p, kv = k.next()
if not p:
break
if 'array_name_a' in kv:
ARRAY_NAME = kv['array_name_a']
else:
LOGGER.error(
"Kef file does not contain entry for array_name_a. "
"Can not continue!")
return False
ref = EX.ph5_g_reports.ph5_t_report
if p not in columns.TABLES:
LOGGER.warning("No table reference for key: {0}. "
"Possibly ph5 file is not open or initialized?"
.format(p))
key = []
errs_keys, errs_required = columns.validate(ref, kv, key)
for e in errs_keys + errs_required:
LOGGER.error(e)
key = None
columns.populate(ref, kv, key)
return True
def write_arrays(Array_t):
''' Write /Experiment_g/Sorts_g/Array_t_xxx '''
def station_cmp(x, y):
return cmp(x['id_s'], y['id_s'])
lines = Array_t.keys()
lines.sort()
# Loop through arrays/lines
for line in lines:
#name = EX.ph5_g_sorts.nextName ()
name = "Array_t_{0:03d}".format(int(line))
a = EX.ph5_g_sorts.newArraySort(name)
stations = Array_t[line].keys()
stations.sort()
Array = {}
n = 0
# Loop through stations
for station in stations:
chan_sets = Array_t[line][station].keys()
chan_sets.sort()
# Loop through channel sets
for chan_set in chan_sets:
try:
for array_t in Array_t[line][station][chan_set]:
columns.populate(a, array_t)
except Exception as e:
print e.message
def write_events(Event_t):
''' Write /Experiment_g/Sorts_g/Event_t '''
keys = sorted(Event_t.keys())
for k in keys:
a = EX.ph5_g_sorts.ph5_t_event
for event_t in Event_t[k]:
columns.populate(a, event_t)
def write_arrays(Array_t):
''' Write /Experiment_g/Sorts_g/Array_t_xxx '''
keys = sorted(Array_t.keys())
for k in keys:
name = EX.ph5_g_sorts.nextName()
a = EX.ph5_g_sorts.newSort(name)
for array_t in Array_t[k]:
columns.populate(a, array_t)
def write_arrays(Array_t):
''' Write /Experiment_g/Sorts_g/Array_t_xxx '''
lines = sorted(Array_t.keys())
# Loop through arrays/lines
for line in lines:
name = "Array_t_{0:03d}".format(int(line))
a = EX.ph5_g_sorts.newArraySort(name)
das_list = sorted(Array_t[line].keys())
# Loop through das_list
for das in das_list:
dtimes = sorted(Array_t[line][das].keys())
# Loop through deploying times
for dtime in dtimes:
chan_sets = sorted(Array_t[line][das][dtime].keys())
# Loop through channel sets
for chan_set in chan_sets:
try:
for array_t in Array_t[line][das][dtime][chan_set]:
columns.populate(a, array_t)
except Exception as e:
print e.message
def toph5(self, parsed_array):
"""
takes a list of dictionaries containing station metadata
and loads them in to PH5
:type list of dictionaries
:param parsed_array
:return:
TODO: Check if data exists in array
"""
sample_rates = []
for entry in parsed_array:
if entry['sample_rate_i'] not in sample_rates:
sample_rates.append(entry['sample_rate_i'])
array_count = 1
# create arrays for each sample rate and assign sample_rate to array
arrays = {}
for sample_rate in sample_rates:
array_name = self.ph5.ph5_g_sorts.nextName()
self.ph5.ph5_g_sorts.newArraySort(array_name)
arrays[sample_rate] = array_name
array_count = array_count + 1
# iterate through parsed_array and add each entry to the correct
# array based on it's sample rate
for entry in parsed_array:
if entry['sample_rate_i'] in arrays:
array_name = "/Experiment_g/Sorts_g/" + arrays[
entry['sample_rate_i']]
ref = columns.TABLES[array_name]
columns.populate(ref, entry, None)
for entry in self.response_t:
ref = columns.TABLES['/Experiment_g/Responses_g/Response_t']
columns.populate(ref, entry, None)
return True
def update():
global EX, ARRAY_NAME, KEF
# There is a bug in batch update that kills kv
k = kef.Kef(KEF)
k.open()
k.read()
#k.batch_update ()
#k.close () ; sys.exit ()
k.rewind()
ARRAY_NAME = None
while 1:
p, kv = k.next()
if not p: break
if kv.has_key('array_name_a'):
ARRAY_NAME = kv['array_name_a']
else:
sys.stderr.write(
"Error: Kef file does not contain entry for array_name_a.\nCan not continue!\n"
)
return False
# XXX We always append XXX
#mo = updateRE.match (p)
ref = EX.ph5_g_reports.ph5_t_report
if not columns.TABLES.has_key(p):
sys.stderr.write("Warning: No table reference for key: %s\n" % p)
sys.stderr.write("Possibly ph5 file is not open or initialized?\n")
key = []
errs_keys, errs_required = columns.validate(ref, kv, key)
for e in errs_keys + errs_required:
sys.stderr.write(e + '\n')
key = None
columns.populate(ref, kv, key)
return True
def load_response(self, backup_path, array_data, input_csv):
"""
Receive:
:param backup_path: path for backup file
:param array_data: list of station based data
:param input_csv: csv file that provide response file names for the
associated das/sensor model, samplerate, samplerate multiplier, gain
Process:
+ Read response files from input.csv to load response data into ph5
+ Create backup for Response_t and array_t(s) and delete them the
tables in ph5 structure
+ From array_data, get a unique list that group same sensor model,
das model, sample rate, sample rate multiplier, gain, bitweight and
n_i with all station entries of those will be turned into a list
Go through the unique list use n_i to get response entry, check if:
. match with response file names in entry: update n_i of station
entries in list
. response entry has no response files filled: fill response file
names, and update n_i of station entries in list
. otherwise, create new response entry increase 1 from max n_i,
update n_i of station entries in list
+ populate response table and array table with updated response
entries and array entries
"""
# use this instead of open ph5 file as table
ph5table = self.ph5.ph5
# load response files from the paths in input.csv
# U in the mode so that \r will be considered as next line
with open(input_csv, "rU") as f:
csv = f.readlines()
loaded_das = []
loaded_sensor = []
for line in csv:
line_list = line.split(",")
if line_list[0] == 'Das Model':
continue
else:
if line_list[5] != "":
name = str(line_list[0] + "_" + line_list[2] + "_" +
line_list[3] + "_" + line_list[4])
name = name.replace(" ", "").translate(None, ',-=.')
if name not in loaded_das:
das_data = self.read_respdata(line_list[5].rstrip())
if das_data is not None:
self.load_respdata(ph5table, name, das_data,
loaded_das)
if len(line_list) >= 6:
if line_list[6] == '\n':
continue
if line_list[1] not in loaded_sensor:
sensor_data = self.read_respdata(line_list[6].rstrip())
if sensor_data is not None:
name = line_list[1].replace(" ", "")
name = name.translate(None, ',-=.')
self.load_respdata(ph5table, name, sensor_data,
loaded_sensor)
if self.skip_update_resp:
# -s flag
LOGGER.info("Skip updating response index in response_t "
"and array_t.")
return
# assign global variables in tabletokef
tabletokef.EX = self.ph5
tabletokef.TABLE_KEY = None
tabletokef.PATH = backup_path
tabletokef.ARRAY_T = {}
if not self.ph5.Response_t:
self.ph5.read_response_t()
response_t = tabletokef.Rows_Keys(self.ph5.Response_t['rows'],
self.ph5.Response_t['keys'])
# backup and delete response_t
write_backup(response_t, '/Experiment_g/Responses_g/Response_t',
'Response_t')
self.ph5.ph5_g_responses.nuke_response_t()
# backup and delete array_t(s)
tabletokef.read_sort_table()
tabletokef.read_sort_arrays()
for a in self.array:
array_name = 'Array_t_%03d' % int(a)
if array_name in tabletokef.ARRAY_T.keys():
write_backup(tabletokef.ARRAY_T[array_name],
'/Experiment_g/Sorts_g/%s' % array_name,
array_name)
self.ph5.ph5_g_sorts.nuke_array_t(int(a))
unique_list = []
for station in array_data:
if station.receiver_n_i not in [None, '']:
station.station_entry[
'receiver_table_n_i'] = station.receiver_n_i
else:
station.station_entry['receiver_table_n_i'] = 0
item = {
'd_model': station.das_model,
's_model': station.sensor_model,
's_rate': str(station.sample_rate),
's_rate_m': str(station.sample_rate_multiplier),
'gain': str(station.gain),
'gain_u': station.gain_units,
'bit_w': str(station.bit_weight),
'bit_w_u': station.bit_weight_units,
'n_i': station.response_n_i,
'station_entry': station.station_entry
}
if item not in unique_list:
unique_list.append(item)
unique_list = group_list_dict(unique_list, ['station_entry'])
"""
update/add entries in self.ph5.Response_t
using update_array() to update station entries of the array
"""
# all_resp: help access response entry through n_i
all_resp = {item['n_i']: item for item in self.ph5.Response_t['rows']}
# max_n_i: used for new response entry added
max_n_i = max(all_resp.keys())
for x in unique_list:
if x['n_i'] == -1:
# no response signal
continue
response_entry = all_resp[x['n_i']]
if self.check_metadata_format(response_entry, x):
# x match with entry created by metadata
continue
filenames = self.get_resp_file_names(x)
if not filenames:
continue
das_resp_name, sen_resp_name = filenames
if response_entry['response_file_das_a'] == '':
# n_i haven't been used, use it
response_entry['response_file_das_a'] = das_resp_name
response_entry['response_file_sensor_a'] = sen_resp_name
self.update_array(x, x['n_i'])
continue
n_i = self.check_filenames(x, all_resp, response_entry, filenames)
if n_i is not None:
# match with n_i created by another resp_load run
self.update_array(x, n_i)
continue
# n_i already used, need new entry
new_response_entry = {}
max_n_i += 1
new_response_entry['n_i'] = max_n_i
new_response_entry['bit_weight/value_d'] = x['bit_w']
new_response_entry['bit_weight/units_s'] = x['bit_w_u']
new_response_entry['gain/value_i'] = x['gain']
new_response_entry['gain/units_s'] = x['gain_u']
new_response_entry['response_file_das_a'] = das_resp_name
new_response_entry['response_file_sensor_a'] = sen_resp_name
new_response_entry['response_file_a'] = ''
self.ph5.Response_t['rows'].append(new_response_entry)
self.update_array(x, max_n_i)
LOGGER.info("%s-%s-%s-%s: n_i %s=>%s" %
(x['s_model'], x['d_model'], x['s_rate'],
x['s_rate_m'], x['n_i'], max_n_i))
# populate response_t with updated entries in self.ph5.Response_t
for entry in self.ph5.Response_t['rows']:
ref = columns.TABLES['/Experiment_g/Responses_g/Response_t']
columns.populate(ref, entry, None)
LOGGER.info("Update Response_t.")
# populate array_t(s) with updated station entries
for a in self.array:
array_name = 'Array_t_%03d' % int(a)
try:
arraybyid = self.ph5.Array_t[array_name]['byid']
arrayorder = self.ph5.Array_t[array_name]['order']
except KeyError:
LOGGER.warning("%s not in ph5." % array_name)
continue
ref = self.ph5.ph5_g_sorts.newArraySort(array_name)
for ph5_station in arrayorder:
station_list = arraybyid.get(ph5_station)
for deployment in station_list:
station_len = len(station_list[deployment])
for st_num in range(0, station_len):
station = station_list[deployment][st_num]
columns.populate(ref, station)
LOGGER.info("Update %s." % array_name)
def single_ts_to_ph5(self, ts_obj, count=1):
"""
load a single time series into ph5
"""
ts_obj.data_logger = ts_obj.data_logger.replace('-', '_')
ts_obj.sampling_rate = int(ts_obj.sampling_rate)
### start populating das table and data arrays
index_t_entry = self.make_index_t_entry(ts_obj)
das_t_entry = self.make_das_entry(ts_obj)
receiver_t_entry = self.make_receiver_entry(ts_obj)
array_t_entry = self.make_array_entry(ts_obj)
sorts_t_entry = self.make_sorts_entry(ts_obj)
receiver_exists, receiver_count = self.get_receiver_n(receiver_t_entry)
### add receiver entry number
das_t_entry['receiver_table_n_i'] = receiver_count
das_t_entry['response_table_n_i'] = count
array_t_entry['receiver_table_n_i'] = receiver_count
### get the current mini file
current_mini = self.get_current_mini_num(ts_obj.data_logger)
mini_handle, mini_name = self.open_mini(current_mini)
current_das_table_mini = self.get_current_das(mini_handle,
ts_obj.data_logger)
mini_handle.ph5_g_receivers.setcurrent(current_das_table_mini)
### make name for array data going into mini file
while True:
next_ = '{0:05}'.format(count)
das_t_entry['array_name_data_a'] = "Data_a_{0}".format(next_)
node = mini_handle.ph5_g_receivers.find_trace_ref(
das_t_entry['array_name_data_a'])
if not node:
break
count = count + 1
continue
### make a new array
mini_handle.ph5_g_receivers.newarray(das_t_entry['array_name_data_a'],
ts_obj.ts.data,
dtype=ts_obj.ts.data.dtype,
description=None)
### create external file names
index_t_entry['external_file_name_s'] = "./{}".format(mini_name)
das_path = "/Experiment_g/Receivers_g/Das_g_{0}".format(
ts_obj.data_logger)
index_t_entry['hdf5_path_s'] = das_path
### populate metadata tables
### DAS goes in both mini and main
mini_handle.ph5_g_receivers.populateDas_t(das_t_entry)
current_das_table_main = self.get_current_das(self.ph5_obj,
ts_obj.data_logger)
self.ph5_obj.ph5_g_receivers.setcurrent(current_das_table_main)
self.ph5_obj.ph5_g_receivers.populateDas_t(das_t_entry)
### index and receivers goes in main
self.ph5_obj.ph5_g_receivers.populateIndex_t(index_t_entry)
if not receiver_exists:
self.ph5_obj.ph5_g_receivers.populateReceiver_t(receiver_t_entry)
#mini_handle.ph5_g_receivers.populateTime_t_()
columns.populate(self.array_table, array_t_entry)
self.ph5_obj.ph5_g_sorts.populateSort_t(sorts_t_entry)
# Don't forget to close minifile
mini_handle.ph5close()
self.logger.info('Loaded {0} to mini file {1}'.format(
ts_obj.fn, mini_name))
return count
def load_response(self, path, nickname, data, input_csv):
ph5 = tables.open_file(os.path.join(path, nickname), "a")
with open(input_csv, "r") as f:
csv = f.readlines()
f.close()
loaded_das = []
loaded_sensor = []
for line in csv:
line_list = line.split(",")
if line_list[0] == 'Das Model':
continue
else:
if line_list[5] != "":
name = str(
line_list[0] +
"_" +
line_list[2] +
"_" +
line_list[3] +
"_" +
line_list[4])
if name not in loaded_das:
with open(line_list[5].rstrip(), "r") as f:
das_data = f.readlines()
f.close()
try:
name = name.replace(
" ", "")
name = name.translate(None, ',-=.')
ph5.create_array(
ph5.root.Experiment_g.Responses_g,
name.replace(
" ", ""), das_data)
loaded_das.append(name)
LOGGER.info(
"Loaded {0}".format(name.replace(" ", "")))
except BaseException:
LOGGER.warning(
"Could not load {0}"
.format(name.replace(" ", "")))
if len(line_list) >= 6:
if line_list[6] == '\n':
continue
if line_list[1] not in loaded_sensor:
with open(line_list[6].rstrip(), "r") as f:
sensor_data = f.readlines()
f.close()
try:
name = line_list[1].replace(
" ", "")
name = name.translate(None, ',-=.')
ph5.create_array(
ph5.root.Experiment_g.Responses_g,
name.replace(
" ", ""), sensor_data)
loaded_sensor.append(line_list[1])
LOGGER.info(
"Loaded {0}"
.format(name.replace(" ", "")))
except BaseException:
LOGGER.warning(
"Could not load {0}"
.format(name.replace(" ", "")))
ph5.close()
ph5_object = experiment.ExperimentGroup(nickname=nickname,
currentpath=path)
ph5_object.ph5open(True)
ph5_object.initgroup()
ret, blah = ph5_object.ph5_g_responses.read_responses()
data_list = []
data_update = []
for station in data:
data_list.append([str(station.das_model),
str(station.sensor_model),
str(station.sample_rate),
str(station.sample_rate_multiplier),
str(station.gain),
str(station.bit_weight),
str(station.bit_weight_units),
str(station.gain_units)])
unique_list = [list(x) for x in set(tuple(x) for x in data_list)]
# get highest n_i
n_i = 0
for entry in ret:
if entry['n_i'] >= n_i:
n_i = entry['n_i']
n_i = n_i + 1
no_resp = list()
for entry in ret:
if (not entry['response_file_das_a']
and not entry['response_file_sensor_a']):
no_resp.append(entry)
has_resp = list()
for entry in ret:
if entry['response_file_das_a'] or entry['response_file_sensor_a']:
has_resp.append(entry)
final_ret = list()
final_ret.extend(has_resp)
for x in unique_list:
response_entry = {}
if x[0]:
name = str(x[0]) + "_" + str(x[2]) + "_" + \
str(x[3]) + "_" + str(x[4])
name_full = '/Experiment_g/' +\
'Responses_g/' + name.replace(" ", "")
name_full = name_full.translate(None, ',-=.')
response_entry['response_file_das_a'] = name_full
if x[1]:
sens = x[1]
sens = sens.translate(None, ',-=.')
name = '/Experiment_g/' +\
'Responses_g/' + sens
response_entry['response_file_sensor_a'] = name
response_entry['bit_weight/value_d'] = x[5]
response_entry['bit_weight/units_s'] = x[6]
response_entry['gain/value_i'] = x[4]
response_entry['gain/units_s'] = x[7]
for resp in no_resp:
if (str(resp['bit_weight/value_d']) ==
str(response_entry['bit_weight/value_d']) and
str(resp['gain/value_i']) ==
str(response_entry['gain/value_i'])):
response_entry['n_i'] = resp['n_i']
final_ret.append(response_entry)
data_update.append([x[0], x[1], x[2], x[3], x[4], n_i])
ph5_object.ph5_g_responses.nuke_response_t()
final_ret.sort()
for entry in final_ret:
ref = columns.TABLES['/Experiment_g/Responses_g/Response_t']
columns.populate(ref, entry, None)
ph5_object.ph5close()
LOGGER.info(
"response_t.kef written into PH5")
for station in data:
for x in data_update:
if station.das_model == x[0] and str(
station.sensor_model) == x[1] and int(
station.sample_rate) == int(
x[2]) and int(
station.sample_rate_multiplier) ==\
int(x[3]) and int(station.gain) == int(x[4]):
station.response_n_i = x[5]
true_sr =\
float(station.sample_rate) /\
float(station.sample_rate_multiplier)
if true_sr < 1.0:
station.response_n_i = None
return data
def batch_update(self, trace=False):
''' Batch update ph5 file from kef file '''
err = False
self.rewind()
p, kv = self.next()
while p:
if trace is True:
kys = kv.keys()
print("=-" * 30)
print("{0}".format(p))
for k in kys:
print("\t{0} = {1}".format(k, kv[k]))
DELETE = False
# Update or Append or Delete
mo = deleteRE.match(p)
if mo:
DELETE = True
else:
mo = updateRE.match(p)
key = []
if mo:
p, k = mo.groups()
key.append(k)
# columns.TABLES keeps a dictionary of key = table name, value =
# reference to table
if p not in columns.TABLES:
LOGGER.warning("No table reference for key: {0}".format(p))
LOGGER.info("Possibly ph5 file is not open or initialized?")
p, kv = self.next()
continue
# Get handle
ref = columns.TABLES[p]
# key needs to be list for columns.validate
if trace is True:
LOGGER.info("Validating...")
errs_keys, errs_required = columns.validate(ref, kv, key)
for e in errs_keys + errs_required:
err = True
LOGGER.error(e)
if trace is True:
LOGGER.info("Done")
if len(key) == 0:
key = None
else:
key = key.pop(0)
if DELETE:
if trace is True:
LOGGER.info("Deleting...")
else:
columns.delete(ref, kv[key], key)
else:
if trace is True:
LOGGER.info("Updating...")
else:
columns.populate(ref, kv, key)
if trace is True:
LOGGER.info("Skipped")
p, kv = self.next()
return err
def batch_update(self, trace=False):
''' Batch update ph5 file from kef file '''
err = False
self.rewind()
p, kv = self.next()
while p:
if trace == True:
kys = kv.keys()
sys.stderr.write('=-' * 30)
sys.stderr.write("\n%s\n" % p)
for k in kys:
sys.stderr.write("\t%s = %s\n" % (k, kv[k]))
DELETE = False
# Update or Append or Delete
mo = deleteRE.match(p)
if mo:
DELETE = True
else:
mo = updateRE.match(p)
key = []
if mo:
p, k = mo.groups()
key.append(k)
#if receiverRE.match (p) :
## We are trying to update something in a Receivers_g
#sys.stderr.write ("Warning: Attempting to modify something under /Experiment_g/Receivers_g.\n")
# columns.TABLES keeps a dictionary of key = table name, value = reference to table
if not columns.TABLES.has_key(p):
sys.stderr.write("Warning: No table reference for key: %s\n" %
p)
sys.stderr.write(
"Possibly ph5 file is not open or initialized?\n")
p, kv = self.next()
continue
# Get handle
ref = columns.TABLES[p]
# key needs to be list for columns.validate
if trace == True:
sys.stderr.write("Validating...\n")
errs_keys, errs_required = columns.validate(ref, kv, key)
for e in errs_keys + errs_required:
err = True
sys.stderr.write(e + '\n')
if trace == True:
sys.stderr.write("Done\n")
if len(key) == 0:
key = None
else:
key = key.pop(0)
if DELETE:
if trace == True:
sys.stderr.write("Deleting...")
else:
columns.delete(ref, kv[key], key)
else:
if trace == True:
sys.stderr.write("Updating...")
else:
columns.populate(ref, kv, key)
if trace == True:
sys.stderr.write("Skipped\n")
p, kv = self.next()
return err