def test_write_backup(self): response_entry = {} response_entry['n_i'] = 1 response_entry['bit_weight/value_d'] = 2 response_entry['gain/value_i'] = 1 response_entry['response_file_das_a'] = "das_path" response_entry['response_file_sensor_a'] = "sens_path" ref = columns.TABLES['/Experiment_g/Responses_g/Response_t'] columns.populate(ref, response_entry, None) self.n_i_fix.ph5.read_response_t() response_t = tabletokef.Rows_Keys(self.n_i_fix.ph5.Response_t['rows'], self.n_i_fix.ph5.Response_t['keys']) with LogCapture() as log: resp_load.write_backup(response_t, '/Experiment_g/Responses_g/Response_t', 'Response_t') msg = log.records[0].msg self.assertEqual(msg[0:33] + msg[40:], 'Writing table backup: Response_t__00.kef.') filename = msg.split(': ')[1][:-1] with open(os.path.join(self.tmpdir, filename), 'r') as content_file: content = content_file.read().strip().split("Table row 1")[1] self.assertEqual( content, "\n/Experiment_g/Responses_g/Response_t" "\n\tn_i=1" "\n\tbit_weight/value_d=2.0" "\n\tbit_weight/units_s=" "\n\tgain/units_s=" "\n\tgain/value_i=1" "\n\tresponse_file_a=" "\n\tresponse_file_das_a=das_path" "\n\tresponse_file_sensor_a=sens_path")
def update(): global EX, ARRAY_NAME, KEF # There is a bug in batch update that kills kv k = kef.Kef(KEF) k.open() k.read() k.rewind() ARRAY_NAME = None while True: p, kv = k.next() if not p: break if 'array_name_a' in kv: ARRAY_NAME = kv['array_name_a'] else: LOGGER.error( "Kef file does not contain entry for array_name_a. " "Can not continue!") return False ref = EX.ph5_g_reports.ph5_t_report if p not in columns.TABLES: LOGGER.warning("No table reference for key: {0}. " "Possibly ph5 file is not open or initialized?" .format(p)) key = [] errs_keys, errs_required = columns.validate(ref, kv, key) for e in errs_keys + errs_required: LOGGER.error(e) key = None columns.populate(ref, kv, key) return True
def write_arrays(Array_t): ''' Write /Experiment_g/Sorts_g/Array_t_xxx ''' def station_cmp(x, y): return cmp(x['id_s'], y['id_s']) lines = Array_t.keys() lines.sort() # Loop through arrays/lines for line in lines: #name = EX.ph5_g_sorts.nextName () name = "Array_t_{0:03d}".format(int(line)) a = EX.ph5_g_sorts.newArraySort(name) stations = Array_t[line].keys() stations.sort() Array = {} n = 0 # Loop through stations for station in stations: chan_sets = Array_t[line][station].keys() chan_sets.sort() # Loop through channel sets for chan_set in chan_sets: try: for array_t in Array_t[line][station][chan_set]: columns.populate(a, array_t) except Exception as e: print e.message
def write_events(Event_t): ''' Write /Experiment_g/Sorts_g/Event_t ''' keys = sorted(Event_t.keys()) for k in keys: a = EX.ph5_g_sorts.ph5_t_event for event_t in Event_t[k]: columns.populate(a, event_t)
def write_arrays(Array_t): ''' Write /Experiment_g/Sorts_g/Array_t_xxx ''' keys = sorted(Array_t.keys()) for k in keys: name = EX.ph5_g_sorts.nextName() a = EX.ph5_g_sorts.newSort(name) for array_t in Array_t[k]: columns.populate(a, array_t)
def write_arrays(Array_t): ''' Write /Experiment_g/Sorts_g/Array_t_xxx ''' lines = sorted(Array_t.keys()) # Loop through arrays/lines for line in lines: name = "Array_t_{0:03d}".format(int(line)) a = EX.ph5_g_sorts.newArraySort(name) das_list = sorted(Array_t[line].keys()) # Loop through das_list for das in das_list: dtimes = sorted(Array_t[line][das].keys()) # Loop through deploying times for dtime in dtimes: chan_sets = sorted(Array_t[line][das][dtime].keys()) # Loop through channel sets for chan_set in chan_sets: try: for array_t in Array_t[line][das][dtime][chan_set]: columns.populate(a, array_t) except Exception as e: print e.message
def toph5(self, parsed_array): """ takes a list of dictionaries containing station metadata and loads them in to PH5 :type list of dictionaries :param parsed_array :return: TODO: Check if data exists in array """ sample_rates = [] for entry in parsed_array: if entry['sample_rate_i'] not in sample_rates: sample_rates.append(entry['sample_rate_i']) array_count = 1 # create arrays for each sample rate and assign sample_rate to array arrays = {} for sample_rate in sample_rates: array_name = self.ph5.ph5_g_sorts.nextName() self.ph5.ph5_g_sorts.newArraySort(array_name) arrays[sample_rate] = array_name array_count = array_count + 1 # iterate through parsed_array and add each entry to the correct # array based on it's sample rate for entry in parsed_array: if entry['sample_rate_i'] in arrays: array_name = "/Experiment_g/Sorts_g/" + arrays[ entry['sample_rate_i']] ref = columns.TABLES[array_name] columns.populate(ref, entry, None) for entry in self.response_t: ref = columns.TABLES['/Experiment_g/Responses_g/Response_t'] columns.populate(ref, entry, None) return True
def update(): global EX, ARRAY_NAME, KEF # There is a bug in batch update that kills kv k = kef.Kef(KEF) k.open() k.read() #k.batch_update () #k.close () ; sys.exit () k.rewind() ARRAY_NAME = None while 1: p, kv = k.next() if not p: break if kv.has_key('array_name_a'): ARRAY_NAME = kv['array_name_a'] else: sys.stderr.write( "Error: Kef file does not contain entry for array_name_a.\nCan not continue!\n" ) return False # XXX We always append XXX #mo = updateRE.match (p) ref = EX.ph5_g_reports.ph5_t_report if not columns.TABLES.has_key(p): sys.stderr.write("Warning: No table reference for key: %s\n" % p) sys.stderr.write("Possibly ph5 file is not open or initialized?\n") key = [] errs_keys, errs_required = columns.validate(ref, kv, key) for e in errs_keys + errs_required: sys.stderr.write(e + '\n') key = None columns.populate(ref, kv, key) return True
def load_response(self, backup_path, array_data, input_csv): """ Receive: :param backup_path: path for backup file :param array_data: list of station based data :param input_csv: csv file that provide response file names for the associated das/sensor model, samplerate, samplerate multiplier, gain Process: + Read response files from input.csv to load response data into ph5 + Create backup for Response_t and array_t(s) and delete them the tables in ph5 structure + From array_data, get a unique list that group same sensor model, das model, sample rate, sample rate multiplier, gain, bitweight and n_i with all station entries of those will be turned into a list Go through the unique list use n_i to get response entry, check if: . match with response file names in entry: update n_i of station entries in list . response entry has no response files filled: fill response file names, and update n_i of station entries in list . otherwise, create new response entry increase 1 from max n_i, update n_i of station entries in list + populate response table and array table with updated response entries and array entries """ # use this instead of open ph5 file as table ph5table = self.ph5.ph5 # load response files from the paths in input.csv # U in the mode so that \r will be considered as next line with open(input_csv, "rU") as f: csv = f.readlines() loaded_das = [] loaded_sensor = [] for line in csv: line_list = line.split(",") if line_list[0] == 'Das Model': continue else: if line_list[5] != "": name = str(line_list[0] + "_" + line_list[2] + "_" + line_list[3] + "_" + line_list[4]) name = name.replace(" ", "").translate(None, ',-=.') if name not in loaded_das: das_data = self.read_respdata(line_list[5].rstrip()) if das_data is not None: self.load_respdata(ph5table, name, das_data, loaded_das) if len(line_list) >= 6: if line_list[6] == '\n': continue if line_list[1] not in loaded_sensor: sensor_data = self.read_respdata(line_list[6].rstrip()) if sensor_data is not None: name = line_list[1].replace(" ", "") name = name.translate(None, ',-=.') self.load_respdata(ph5table, name, sensor_data, loaded_sensor) if self.skip_update_resp: # -s flag LOGGER.info("Skip updating response index in response_t " "and array_t.") return # assign global variables in tabletokef tabletokef.EX = self.ph5 tabletokef.TABLE_KEY = None tabletokef.PATH = backup_path tabletokef.ARRAY_T = {} if not self.ph5.Response_t: self.ph5.read_response_t() response_t = tabletokef.Rows_Keys(self.ph5.Response_t['rows'], self.ph5.Response_t['keys']) # backup and delete response_t write_backup(response_t, '/Experiment_g/Responses_g/Response_t', 'Response_t') self.ph5.ph5_g_responses.nuke_response_t() # backup and delete array_t(s) tabletokef.read_sort_table() tabletokef.read_sort_arrays() for a in self.array: array_name = 'Array_t_%03d' % int(a) if array_name in tabletokef.ARRAY_T.keys(): write_backup(tabletokef.ARRAY_T[array_name], '/Experiment_g/Sorts_g/%s' % array_name, array_name) self.ph5.ph5_g_sorts.nuke_array_t(int(a)) unique_list = [] for station in array_data: if station.receiver_n_i not in [None, '']: station.station_entry[ 'receiver_table_n_i'] = station.receiver_n_i else: station.station_entry['receiver_table_n_i'] = 0 item = { 'd_model': station.das_model, 's_model': station.sensor_model, 's_rate': str(station.sample_rate), 's_rate_m': str(station.sample_rate_multiplier), 'gain': str(station.gain), 'gain_u': station.gain_units, 'bit_w': str(station.bit_weight), 'bit_w_u': station.bit_weight_units, 'n_i': station.response_n_i, 'station_entry': station.station_entry } if item not in unique_list: unique_list.append(item) unique_list = group_list_dict(unique_list, ['station_entry']) """ update/add entries in self.ph5.Response_t using update_array() to update station entries of the array """ # all_resp: help access response entry through n_i all_resp = {item['n_i']: item for item in self.ph5.Response_t['rows']} # max_n_i: used for new response entry added max_n_i = max(all_resp.keys()) for x in unique_list: if x['n_i'] == -1: # no response signal continue response_entry = all_resp[x['n_i']] if self.check_metadata_format(response_entry, x): # x match with entry created by metadata continue filenames = self.get_resp_file_names(x) if not filenames: continue das_resp_name, sen_resp_name = filenames if response_entry['response_file_das_a'] == '': # n_i haven't been used, use it response_entry['response_file_das_a'] = das_resp_name response_entry['response_file_sensor_a'] = sen_resp_name self.update_array(x, x['n_i']) continue n_i = self.check_filenames(x, all_resp, response_entry, filenames) if n_i is not None: # match with n_i created by another resp_load run self.update_array(x, n_i) continue # n_i already used, need new entry new_response_entry = {} max_n_i += 1 new_response_entry['n_i'] = max_n_i new_response_entry['bit_weight/value_d'] = x['bit_w'] new_response_entry['bit_weight/units_s'] = x['bit_w_u'] new_response_entry['gain/value_i'] = x['gain'] new_response_entry['gain/units_s'] = x['gain_u'] new_response_entry['response_file_das_a'] = das_resp_name new_response_entry['response_file_sensor_a'] = sen_resp_name new_response_entry['response_file_a'] = '' self.ph5.Response_t['rows'].append(new_response_entry) self.update_array(x, max_n_i) LOGGER.info("%s-%s-%s-%s: n_i %s=>%s" % (x['s_model'], x['d_model'], x['s_rate'], x['s_rate_m'], x['n_i'], max_n_i)) # populate response_t with updated entries in self.ph5.Response_t for entry in self.ph5.Response_t['rows']: ref = columns.TABLES['/Experiment_g/Responses_g/Response_t'] columns.populate(ref, entry, None) LOGGER.info("Update Response_t.") # populate array_t(s) with updated station entries for a in self.array: array_name = 'Array_t_%03d' % int(a) try: arraybyid = self.ph5.Array_t[array_name]['byid'] arrayorder = self.ph5.Array_t[array_name]['order'] except KeyError: LOGGER.warning("%s not in ph5." % array_name) continue ref = self.ph5.ph5_g_sorts.newArraySort(array_name) for ph5_station in arrayorder: station_list = arraybyid.get(ph5_station) for deployment in station_list: station_len = len(station_list[deployment]) for st_num in range(0, station_len): station = station_list[deployment][st_num] columns.populate(ref, station) LOGGER.info("Update %s." % array_name)
def single_ts_to_ph5(self, ts_obj, count=1): """ load a single time series into ph5 """ ts_obj.data_logger = ts_obj.data_logger.replace('-', '_') ts_obj.sampling_rate = int(ts_obj.sampling_rate) ### start populating das table and data arrays index_t_entry = self.make_index_t_entry(ts_obj) das_t_entry = self.make_das_entry(ts_obj) receiver_t_entry = self.make_receiver_entry(ts_obj) array_t_entry = self.make_array_entry(ts_obj) sorts_t_entry = self.make_sorts_entry(ts_obj) receiver_exists, receiver_count = self.get_receiver_n(receiver_t_entry) ### add receiver entry number das_t_entry['receiver_table_n_i'] = receiver_count das_t_entry['response_table_n_i'] = count array_t_entry['receiver_table_n_i'] = receiver_count ### get the current mini file current_mini = self.get_current_mini_num(ts_obj.data_logger) mini_handle, mini_name = self.open_mini(current_mini) current_das_table_mini = self.get_current_das(mini_handle, ts_obj.data_logger) mini_handle.ph5_g_receivers.setcurrent(current_das_table_mini) ### make name for array data going into mini file while True: next_ = '{0:05}'.format(count) das_t_entry['array_name_data_a'] = "Data_a_{0}".format(next_) node = mini_handle.ph5_g_receivers.find_trace_ref( das_t_entry['array_name_data_a']) if not node: break count = count + 1 continue ### make a new array mini_handle.ph5_g_receivers.newarray(das_t_entry['array_name_data_a'], ts_obj.ts.data, dtype=ts_obj.ts.data.dtype, description=None) ### create external file names index_t_entry['external_file_name_s'] = "./{}".format(mini_name) das_path = "/Experiment_g/Receivers_g/Das_g_{0}".format( ts_obj.data_logger) index_t_entry['hdf5_path_s'] = das_path ### populate metadata tables ### DAS goes in both mini and main mini_handle.ph5_g_receivers.populateDas_t(das_t_entry) current_das_table_main = self.get_current_das(self.ph5_obj, ts_obj.data_logger) self.ph5_obj.ph5_g_receivers.setcurrent(current_das_table_main) self.ph5_obj.ph5_g_receivers.populateDas_t(das_t_entry) ### index and receivers goes in main self.ph5_obj.ph5_g_receivers.populateIndex_t(index_t_entry) if not receiver_exists: self.ph5_obj.ph5_g_receivers.populateReceiver_t(receiver_t_entry) #mini_handle.ph5_g_receivers.populateTime_t_() columns.populate(self.array_table, array_t_entry) self.ph5_obj.ph5_g_sorts.populateSort_t(sorts_t_entry) # Don't forget to close minifile mini_handle.ph5close() self.logger.info('Loaded {0} to mini file {1}'.format( ts_obj.fn, mini_name)) return count
def load_response(self, path, nickname, data, input_csv): ph5 = tables.open_file(os.path.join(path, nickname), "a") with open(input_csv, "r") as f: csv = f.readlines() f.close() loaded_das = [] loaded_sensor = [] for line in csv: line_list = line.split(",") if line_list[0] == 'Das Model': continue else: if line_list[5] != "": name = str( line_list[0] + "_" + line_list[2] + "_" + line_list[3] + "_" + line_list[4]) if name not in loaded_das: with open(line_list[5].rstrip(), "r") as f: das_data = f.readlines() f.close() try: name = name.replace( " ", "") name = name.translate(None, ',-=.') ph5.create_array( ph5.root.Experiment_g.Responses_g, name.replace( " ", ""), das_data) loaded_das.append(name) LOGGER.info( "Loaded {0}".format(name.replace(" ", ""))) except BaseException: LOGGER.warning( "Could not load {0}" .format(name.replace(" ", ""))) if len(line_list) >= 6: if line_list[6] == '\n': continue if line_list[1] not in loaded_sensor: with open(line_list[6].rstrip(), "r") as f: sensor_data = f.readlines() f.close() try: name = line_list[1].replace( " ", "") name = name.translate(None, ',-=.') ph5.create_array( ph5.root.Experiment_g.Responses_g, name.replace( " ", ""), sensor_data) loaded_sensor.append(line_list[1]) LOGGER.info( "Loaded {0}" .format(name.replace(" ", ""))) except BaseException: LOGGER.warning( "Could not load {0}" .format(name.replace(" ", ""))) ph5.close() ph5_object = experiment.ExperimentGroup(nickname=nickname, currentpath=path) ph5_object.ph5open(True) ph5_object.initgroup() ret, blah = ph5_object.ph5_g_responses.read_responses() data_list = [] data_update = [] for station in data: data_list.append([str(station.das_model), str(station.sensor_model), str(station.sample_rate), str(station.sample_rate_multiplier), str(station.gain), str(station.bit_weight), str(station.bit_weight_units), str(station.gain_units)]) unique_list = [list(x) for x in set(tuple(x) for x in data_list)] # get highest n_i n_i = 0 for entry in ret: if entry['n_i'] >= n_i: n_i = entry['n_i'] n_i = n_i + 1 no_resp = list() for entry in ret: if (not entry['response_file_das_a'] and not entry['response_file_sensor_a']): no_resp.append(entry) has_resp = list() for entry in ret: if entry['response_file_das_a'] or entry['response_file_sensor_a']: has_resp.append(entry) final_ret = list() final_ret.extend(has_resp) for x in unique_list: response_entry = {} if x[0]: name = str(x[0]) + "_" + str(x[2]) + "_" + \ str(x[3]) + "_" + str(x[4]) name_full = '/Experiment_g/' +\ 'Responses_g/' + name.replace(" ", "") name_full = name_full.translate(None, ',-=.') response_entry['response_file_das_a'] = name_full if x[1]: sens = x[1] sens = sens.translate(None, ',-=.') name = '/Experiment_g/' +\ 'Responses_g/' + sens response_entry['response_file_sensor_a'] = name response_entry['bit_weight/value_d'] = x[5] response_entry['bit_weight/units_s'] = x[6] response_entry['gain/value_i'] = x[4] response_entry['gain/units_s'] = x[7] for resp in no_resp: if (str(resp['bit_weight/value_d']) == str(response_entry['bit_weight/value_d']) and str(resp['gain/value_i']) == str(response_entry['gain/value_i'])): response_entry['n_i'] = resp['n_i'] final_ret.append(response_entry) data_update.append([x[0], x[1], x[2], x[3], x[4], n_i]) ph5_object.ph5_g_responses.nuke_response_t() final_ret.sort() for entry in final_ret: ref = columns.TABLES['/Experiment_g/Responses_g/Response_t'] columns.populate(ref, entry, None) ph5_object.ph5close() LOGGER.info( "response_t.kef written into PH5") for station in data: for x in data_update: if station.das_model == x[0] and str( station.sensor_model) == x[1] and int( station.sample_rate) == int( x[2]) and int( station.sample_rate_multiplier) ==\ int(x[3]) and int(station.gain) == int(x[4]): station.response_n_i = x[5] true_sr =\ float(station.sample_rate) /\ float(station.sample_rate_multiplier) if true_sr < 1.0: station.response_n_i = None return data
def batch_update(self, trace=False): ''' Batch update ph5 file from kef file ''' err = False self.rewind() p, kv = self.next() while p: if trace is True: kys = kv.keys() print("=-" * 30) print("{0}".format(p)) for k in kys: print("\t{0} = {1}".format(k, kv[k])) DELETE = False # Update or Append or Delete mo = deleteRE.match(p) if mo: DELETE = True else: mo = updateRE.match(p) key = [] if mo: p, k = mo.groups() key.append(k) # columns.TABLES keeps a dictionary of key = table name, value = # reference to table if p not in columns.TABLES: LOGGER.warning("No table reference for key: {0}".format(p)) LOGGER.info("Possibly ph5 file is not open or initialized?") p, kv = self.next() continue # Get handle ref = columns.TABLES[p] # key needs to be list for columns.validate if trace is True: LOGGER.info("Validating...") errs_keys, errs_required = columns.validate(ref, kv, key) for e in errs_keys + errs_required: err = True LOGGER.error(e) if trace is True: LOGGER.info("Done") if len(key) == 0: key = None else: key = key.pop(0) if DELETE: if trace is True: LOGGER.info("Deleting...") else: columns.delete(ref, kv[key], key) else: if trace is True: LOGGER.info("Updating...") else: columns.populate(ref, kv, key) if trace is True: LOGGER.info("Skipped") p, kv = self.next() return err
def batch_update(self, trace=False): ''' Batch update ph5 file from kef file ''' err = False self.rewind() p, kv = self.next() while p: if trace == True: kys = kv.keys() sys.stderr.write('=-' * 30) sys.stderr.write("\n%s\n" % p) for k in kys: sys.stderr.write("\t%s = %s\n" % (k, kv[k])) DELETE = False # Update or Append or Delete mo = deleteRE.match(p) if mo: DELETE = True else: mo = updateRE.match(p) key = [] if mo: p, k = mo.groups() key.append(k) #if receiverRE.match (p) : ## We are trying to update something in a Receivers_g #sys.stderr.write ("Warning: Attempting to modify something under /Experiment_g/Receivers_g.\n") # columns.TABLES keeps a dictionary of key = table name, value = reference to table if not columns.TABLES.has_key(p): sys.stderr.write("Warning: No table reference for key: %s\n" % p) sys.stderr.write( "Possibly ph5 file is not open or initialized?\n") p, kv = self.next() continue # Get handle ref = columns.TABLES[p] # key needs to be list for columns.validate if trace == True: sys.stderr.write("Validating...\n") errs_keys, errs_required = columns.validate(ref, kv, key) for e in errs_keys + errs_required: err = True sys.stderr.write(e + '\n') if trace == True: sys.stderr.write("Done\n") if len(key) == 0: key = None else: key = key.pop(0) if DELETE: if trace == True: sys.stderr.write("Deleting...") else: columns.delete(ref, kv[key], key) else: if trace == True: sys.stderr.write("Updating...") else: columns.populate(ref, kv, key) if trace == True: sys.stderr.write("Skipped\n") p, kv = self.next() return err