def qc_fields(): ''' Check each column for regular expression, type, range. ''' n = 0 ret = [] for t in TABLE: n += 1 for i in range(len(t)): key = NAMES[i] if key == 'Ignore': continue v = t[i] if 'type' in COLS[key]: ttype = COLS[key]['type'] else: ttype = None if 'help' in COLS[key]: hhelp = COLS[key]['help'] else: hhelp = None if 're' in COLS[key]: rre = COLS[key]['re'] else: rre = None if 'range' in COLS[key]: rrange = COLS[key]['range'] else: rrange = None # Check regular expression if not match_re(v, rre): ret.append( "{0}: Value of column {1} {2} does not match re. Help:" " {3}".format(n, key, v, hhelp)) # Check type if not match_type(v, ttype): ret.append( "{0}: Value of column {1} {2} does not match type. Type:" " {3}".format(n, key, v, ttype)) # Check range if not match_range(v, rrange): ret.append( "{0}: Value of column {1} {2} does not match expected" " range. Range: {3}".format(n, key, v, rrange)) # Check if ascii time if timeRE.match(key): try: timedoy.fdsn2epoch(v, fepoch=True) except timedoy.TimeError: try: timedoy.passcal2epoch(v, fepoch=True) except timedoy.TimeError: ret.append( "{0}: Value of column {1} {2} does not match" " expected time string".format(n, key, v)) return ret
def _is_in(das, shot_time, length, si): ''' Test to see if data is available for a given das starting at shot_time for length. Check to see if there are any gaps greater than the sample interval. Inputs: das - The das serial number as a string shot_time - The shot time as a timedoy object length - Length is seconds as a float si - Sample interval in seconds Returns: A tuple containing: Match first sample time as a timedoy object, None if no match Match last sample time as a timedoy object, None if no match Gaps as a list of start and end times as timedoy objects ''' if DATA.has_key(das): data = DATA[das] else: data = [] shot_start_epoch = shot_time.epoch(fepoch=True) shot_stop_epoch = shot_start_epoch + length hits = [] gaps = [] for d in data: data_start_epoch = d['window_start'].epoch(fepoch=True) data_stop_epoch = d['window_stop'].epoch(fepoch=True) # Call ph5api.is_in if is_in(data_start_epoch, data_stop_epoch, shot_start_epoch, shot_stop_epoch): hits.append(d) # Match no gaps if len(hits) == 1: return hits[0]['first_sample'], hits[0]['last_sample'], gaps # No match elif len(hits) == 0: return None, None, gaps # Match with gaps else: fs = None ls = None for h in hits: if fs == None: fs = h['first_sample'] if ls == None: ls = h['last_sample'] delta = abs( timedoy.passcal2epoch(h['last_sample']) - timedoy.passcal2epoch(ls)) if delta > si: gaps.append((ls, h['last_sample'])) ls = h['last_sample'] return fs, ls, gaps
def qc_time(t1, t2): ''' Measure difference between deployment and pickup times. ''' try: e1 = timedoy.fdsn2epoch(t1) except timedoy.TimeError: e1 = timedoy.passcal2epoch(t1) try: e2 = timedoy.fdsn2epoch(t2) except timedoy.TimeError: e2 = timedoy.passcal2epoch(t2) return e2 - e1
def get_times(key, value): ''' Create time entries for Array_t_xxx or Event_t[_xxx] ''' try: fepoch = timedoy.fdsn2epoch(value, fepoch=True) except timedoy.TimeError: try: fepoch = timedoy.passcal2epoch(value, fepoch=True) except timedoy.TimeError: # This SHOULD never happen pre = key.split('/')[0] LOGGER.error("Bad time value for {0} {1}.".format(key, value)) line = "\t{0}/ascii_s = {1}\n".format(pre, time.ctime(int(0))) line += "\t{0}/epoch_l = {1}\n".format(pre, int(0)) line += "\t{0}/micro_seconds_i = {1}\n".format( pre, int(0. * 1000000.)) line += "\t{0}/type_s = {1}\n".format(pre, 'BOTH') return line f, i = math.modf(fepoch) pre = key.split('/')[0] line = "\t{0}/ascii_s = {1}\n".format(pre, time.ctime(int(i))) line += "\t{0}/epoch_l = {1}\n".format(pre, int(i)) line += "\t{0}/micro_seconds_i = {1}\n".format(pre, int(f * 1000000.)) line += "\t{0}/type_s = {1}\n".format(pre, 'BOTH') return line
def doy_breakup(start_fepoch, length=86400): """ Given a start time epoch returns a next days equivalent epoch time and the difference in seconds between the start and stop epoch times. :param: start_fepoch :type: float :returns: stop_fepoch : next days stop epoch :type: float seconds: difference in seconds between the start and end epoch times :type: float """ passcal_start = epoch2passcal(float(start_fepoch)) start_passcal_list = passcal_start.split(":") start_year = start_passcal_list[0] start_doy = start_passcal_list[1] start_hour = start_passcal_list[2] start_minute = start_passcal_list[3] start_second = start_passcal_list[4] datestr = "{0}:{1}:{2}:{3}:{4}".format(start_year, start_doy, start_hour, start_minute, start_second) passcal_date = datetime.strptime(datestr, "%Y:%j:%H:%M:%S.%f") next_passcal_date = passcal_date + timedelta(seconds=length) next_passcal_date_str = next_passcal_date.strftime("%Y:%j:%H:%M:%S.%f") stop_fepoch = passcal2epoch(next_passcal_date_str, fepoch=True) seconds = stop_fepoch - start_fepoch return stop_fepoch, seconds
def process_all(): ''' Process through each shot line, shot, array, station, component (channel) and print matches to stdout ''' Events = EVENT['Events'] for Event in Events: shot_line = Event['shot_line'] shot_line_name = "Event_t_{0:03d}".format(int(shot_line)) for event in Event['Events']: yr, doy, hr, mn, sc = event['time'].split(':') shot_time = timedoy.TimeDOY(year=int(yr), doy=int(doy), hour=int(hr), minute=int(mn), second=float(sc)) if ARGS.offset_secs: shot_time = shot_time + ARGS.offset_secs shot_id = event['id'] Arrays = ARRAY['Arrays'] for Array in Arrays: array_name = "Array_t_{0:03d}".format(int(Array['array'])) sample_rate = Array['sample_rate'] length = 65536. / sample_rate cut_end = shot_time + length for station in Array['Stations']: chan = station['chan'] das = station['das'] station_id = station['id'] seed_id = station['seed_station_name'] fs, ls, gaps = _is_in(das, shot_time, length, 1. / sample_rate) if fs is None: fs = 'NA' if ls is None: ls = 'NA' if ARGS.epoch: if fs != 'NA': fs = str(timedoy.passcal2epoch(fs, fepoch=True)) if ls != 'NA': ls = str(timedoy.passcal2epoch(ls, fepoch=True)) line = [ shot_line_name, shot_id, str(shot_time.epoch(fepoch=True)), str(cut_end.epoch(fepoch=True)), array_name, station_id, seed_id, das, str(chan), fs, ls ] else: line = [ shot_line_name, shot_id, shot_time.getPasscalTime(ms=True), cut_end.getPasscalTime(ms=True), array_name, station_id, seed_id, das, str(chan), fs, ls ] if ARGS.csv: print ','.join(line) else: print ' '.join(line) if len(gaps) != 0: for g in gaps: print "\t", g[0], g[1]
def create_cut(self, seed_network, ph5_station, seed_station, station_cut_times, station_list, deployment, st_num, array_code, experiment_id): deploy = station_list[deployment][st_num]['deploy_time/epoch_l'] deploy_micro = station_list[deployment][ st_num]['deploy_time/micro_seconds_i'] pickup = station_list[deployment][st_num]['pickup_time/epoch_l'] pickup_micro = station_list[deployment][ st_num]['pickup_time/micro_seconds_i'] location = station_list[deployment][ st_num]['seed_location_code_s'] das = station_list[deployment][st_num]['das/serial_number_s'] das_manufacturer = station_list[deployment][st_num][ 'das/manufacturer_s'] das_model = station_list[deployment][st_num][ 'das/model_s'] sensor_type = " ".join([x for x in [station_list[deployment][st_num][ 'sensor/manufacturer_s'], station_list[deployment][st_num][ 'sensor/model_s']] if x]) receiver_n_i = station_list[deployment][st_num]['receiver_table_n_i'] response_n_i = station_list[deployment][st_num]['response_table_n_i'] if 'sample_rate_i' in station_list[deployment][0]: sample_rate = station_list[deployment][st_num]['sample_rate_i'] sample_rate_multiplier = 1 if ('sample_rate_multiplier_i' in station_list[deployment][st_num]): sample_rate_multiplier = station_list[ deployment][st_num]['sample_rate_multiplier_i'] if self.sample_rate_list: sample_list = self.sample_rate_list if not ph5utils.does_pattern_exists(sample_list, sample_rate): return seed_channel, component = self.get_channel_and_component( station_list, deployment, st_num) if self.component: component_list = self.component if not ph5utils.does_pattern_exists(component_list, component): return if self.channel: cha_patterns = self.channel if not ph5utils.does_pattern_exists(cha_patterns, seed_channel): return if self.das_sn and self.das_sn != das: return if self.reqtype == "FDSN": # trim user defined time range if it extends beyond the # deploy/pickup times if self.start_time: if "T" not in self.start_time: check_start_time = passcal2epoch( self.start_time, fepoch=True) if float(check_start_time) > float(deploy): start_fepoch = self.start_time sct = StationCutTime( passcal2epoch(start_fepoch, fepoch=True) ) station_cut_times.append(sct) else: sct = StationCutTime(deploy) station_cut_times.append(sct) else: check_start_time = ph5utils.datestring_to_epoch( self.start_time) if float(check_start_time) > float(deploy): sct = StationCutTime( ph5utils.datestring_to_epoch(self.start_time)) station_cut_times.append(sct) else: sct = StationCutTime(deploy) station_cut_times.append(sct) if float(check_start_time) > float(pickup): return else: sct = StationCutTime( ph5api.fepoch(deploy, deploy_micro) ) station_cut_times.append(sct) for sct in station_cut_times: start_fepoch = sct.time if self.reqtype == "SHOT" or self.reqtype == "RECEIVER": if self.offset: # adjust starttime by an offset start_fepoch += int(self.offset) if self.length: stop_fepoch = start_fepoch + self.length else: raise PH5toMSAPIError( "Error - length is required for request by shot.") elif self.reqtype == "FDSN": if self.end_time: if "T" not in self.end_time: check_end_time = passcal2epoch( self.end_time, fepoch=True) if float(check_end_time) < float(pickup): stop_fepoch = self.end_time stop_fepoch = passcal2epoch( stop_fepoch, fepoch=True) else: stop_fepoch = pickup else: check_end_time = ph5utils.datestring_to_epoch( self.end_time) if float(check_end_time) < float(pickup): stop_fepoch = ph5utils.datestring_to_epoch( self.end_time) else: stop_fepoch = pickup if float(check_end_time) < float(deploy): continue elif self.length: stop_fepoch = start_fepoch + self.length else: stop_fepoch = ph5api.fepoch(pickup, pickup_micro) if (self.use_deploy_pickup is True and not ((int(start_fepoch) >= deploy and int(stop_fepoch) <= pickup))): # das not deployed within deploy/pickup time continue start_passcal = epoch2passcal(start_fepoch, sep=':') start_passcal_list = start_passcal.split(":") start_doy = start_passcal_list[1] if self.doy_keep: if start_doy not in self.doy: continue midnight_fepoch, secondLeftInday = \ ph5utils.inday_breakup(start_fepoch) # if (stop_fepoch - start_fepoch) > 86400: if (stop_fepoch - start_fepoch) > secondLeftInday: seconds_covered = 0 total_seconds = stop_fepoch - start_fepoch times_to_cut = [] if self.cut_len != 86400: stop_time, seconds = ph5utils.doy_breakup( start_fepoch, self.cut_len) else: stop_time, seconds = ph5utils.inday_breakup(start_fepoch) seconds_covered = seconds_covered + seconds times_to_cut.append([start_fepoch, stop_time]) start_time = stop_time while seconds_covered < total_seconds: if self.cut_len != 86400: stop_time, seconds = ph5utils.doy_breakup( start_time, self.cut_len) else: stop_time, seconds = ph5utils.inday_breakup(start_time) seconds_covered += seconds if stop_time > stop_fepoch: times_to_cut.append([start_time, stop_fepoch]) break times_to_cut.append([start_time, stop_time]) start_time = stop_time else: times_to_cut = [[start_fepoch, stop_fepoch]] times_to_cut[-1][-1] = stop_fepoch latitude = station_list[deployment][ st_num]['location/Y/value_d'] longitude = station_list[deployment][ st_num]['location/X/value_d'] elev = station_list[deployment][ st_num]['location/Z/value_d'] for starttime, endtime in tuple(times_to_cut): try: self.ph5.query_das_t(das, component, starttime, endtime, sample_rate, sample_rate_multiplier ) except experiment.HDF5InteractionError: continue station_x = StationCut( seed_network, experiment_id, ph5_station, seed_station, array_code, das, das_manufacturer, das_model, sensor_type, component, seed_channel, starttime, endtime, sample_rate, sample_rate_multiplier, self.notimecorrect, location, latitude, longitude, elev, receiver_n_i, response_n_i, shot_id=sct.shot_id, shot_lat=sct.shot_lat, shot_lng=sct.shot_lng, shot_elevation=sct.shot_elevation) station_hash = hash(frozenset([seed_station, das, latitude, longitude, sample_rate, sample_rate_multiplier, starttime, endtime])) if station_hash in self.hash_list: continue else: self.hash_list.append(station_hash) yield station_x
def get_args(): ''' Read command line argments ''' global ARGS, P5 import argparse parser = argparse.ArgumentParser() parser.usage = "Version: %s\n" % PROG_VERSION parser.usage += "ph5toevt --eventnumber=shot --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n" parser.usage += "\toptions:\n\t--array=array, --offset=seconds (float), --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n" parser.usage += "ph5toevt --allevents --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n" parser.usage += "\toptions:\n\t--array=array, --offset=seconds (float), --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n" parser.usage += "ph5toevt --starttime=yyyy:jjj:hh:mm:ss[:.]sss --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n" parser.usage += "\toptions:\n\t--stoptime=yyyy:jjj:hh:mm:ss[:.]sss, --array=array, --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n" #parser.usage += "ph5toseg --all, --nickname=experiment_nickname [--path=ph5_directory_path] [--das=das_sn] [--station=station_id] [--doy=comma seperated doy list] [options]" parser.usage += "\n\n\tgeneral options:\n\t--channel=[1,2,3]\n\t--sample_rate_keep=sample_rate\n\t--notimecorrect\n\t--decimation=[2,4,5,8,10,20]\n\t--out_dir=output_directory" parser.description = "Generate SEG-Y gathers in shot order..." # Usually master.ph5 parser.add_argument("-n", "--nickname", dest="ph5_file_prefix", help="The ph5 file prefix (experiment nickname).", metavar="ph5_file_prefix", required=True) # Path to the directory that holds master.ph5 parser.add_argument( "-p", "--path", dest="ph5_path", help="Path to ph5 files. Defaults to current directory.", metavar="ph5_path", default='.') # SEED channel parser.add_argument("--channel", dest="seed_channel", help="Filter on SEED channel.", metavar="seed_channel") # SEED network code parser.add_argument("--network", dest="seed_network", help="Filter on SEED net code.", metavar="seed_network") # SEED loc code parser.add_argument("--location", dest="seed_location", help="Filter on SEED loc code.", metavar="seed_location") # Channels. Will extract in order listed here. 'Usually' 1 -> Z, 2-> N, 3 -> E parser.add_argument( "-c", "--channels", action="store", help="List of comma seperated channels to extract. Default = 1,2,3.", type=str, dest="channels", metavar="channels", default='1,2,3') # Extract a single event parser.add_argument("-e", "--eventnumber", action="store", dest="event_number", type=int, metavar="event_number") # Event id's in order, comma seperated parser.add_argument( "--event_list", dest="evt_list", help= "Comma separated list of event id's to gather from defined or selected events.", metavar="evt_list") # Extract all events in Event_t parser.add_argument("-E", "--allevents", action="store_true", dest="all_events", help="Extract all events in event table.", default=False) # The shot line number, 0 for Event_t parser.add_argument("--shot_line", dest="shot_line", action="store", help="The shot line number that holds the shots.", type=int, metavar="shot_line") # External shot line file parser.add_argument( "--shot_file", dest="shot_file", action="store", help= "Input an external kef file that contains event information, Event_t.kef.", type=str, metavar="shot_file") # Extract data for all stations starting at this time parser.add_argument("-s", "--starttime", action="store", dest="start_time", type=str, metavar="start_time") # The array number parser.add_argument("-A", "--station_array", dest="station_array", action="store", help="The array number that holds the station(s).", type=int, metavar="station_array", required=True) # Length of traces to put in gather parser.add_argument("-l", "--length", action="store", required=True, type=int, dest="length", metavar="length") # Start trace at time offset from shot time parser.add_argument( "-O", "--seconds_offset_from_shot", "--offset", metavar="seconds_offset_from_shot", help="Time in seconds from shot time to start the trace.", type=float, default=0.) # Do not time correct texan data parser.add_argument("-N", "--notimecorrect", action="store_false", default=True, dest="do_time_correct") # Output directory parser.add_argument("-o", "--out_dir", action="store", dest="out_dir", metavar="out_dir", type=str, default=".") # Write to stdout parser.add_argument("--stream", action="store_true", dest="write_stdout", help="Write to stdout instead of a file.", default=False) # Use deploy and pickup times to determine where an instrument was deployed parser.add_argument( "--use_deploy_pickup", action="store_true", default=False, help= "Use deploy and pickup times to determine if data exists for a station.", dest="deploy_pickup") # Stations to gather, comma seperated parser.add_argument( "-S", "--stations", "--station_list", dest="stations_to_gather", help="Comma separated list of stations to receiver gather.", metavar="stations_to_gather", required=False) # Filter out all sample rates except the listed parser.add_argument("-r", "--sample_rate_keep", action="store", dest="sample_rate", metavar="sample_rate", type=float) # Apply a reduction velocity, km parser.add_argument("-V", "--reduction_velocity", action="store", dest="red_vel", help="Reduction velocity in km/sec.", metavar="red_vel", type=float, default="-1.") # Decimate data. Decimation factor parser.add_argument("-d", "--decimation", action="store", choices=["2", "4", "5", "8", "10", "20"], dest="decimation", metavar="decimation") # Convert geographic coordinated in ph5 to UTM before creating gather parser.add_argument("-U", "--UTM", action="store_true", dest="use_utm", help="Fill SEG-Y headers with UTM instead of lat/lon.", default=False) # How to fill in the extended trace header parser.add_argument("-x", "--extended_header", action="store", dest="ext_header", help="Extended trace header style: \ 'P' -> PASSCAL, \ 'S' -> SEG, \ 'U' -> Menlo USGS, \ default = U", choices=["P", "S", "U", "I", "N"], default="U", metavar="extended_header_style") # Ignore channel in Das_t. Only useful with texans parser.add_argument("--ic", action="store_true", dest="ignore_channel", default=False) # Allow traces to be 2^16 samples long vs 2^15 parser.add_argument("--break_standard", action="store_false", dest="break_standard", help="Force traces to be no longer than 2^15 samples.", default=True) parser.add_argument("--debug", dest="debug", action="store_true", default=False) ARGS = parser.parse_args() #print ARGS try: P5 = ph5api.PH5(path=ARGS.ph5_path, nickname=ARGS.ph5_file_prefix) except Exception as e: sys.stderr.write("Error: Can't open {0} at {1}.".format( ARGS.ph5_file_prefix, ARGS.ph5_path)) sys.exit(-1) # if ARGS.shot_file: if not ARGS.shot_line: sys.stderr.write( "Error: Shot line switch, --shot_line, required when using external shot file." ) sys.exit(-3) external = external_file.External(ARGS.shot_file) ARGS.shot_file = external.Event_t P5.Event_t_names = ARGS.shot_file.keys() else: P5.read_event_t_names() # if ARGS.event_number: ARGS.evt_list = list([str(ARGS.event_number)]) elif ARGS.evt_list: ARGS.evt_list = map(str, ARGS.evt_list.split(',')) elif ARGS.start_time: ARGS.start_time = timedoy.TimeDOY( epoch=timedoy.passcal2epoch(ARGS.start_time, fepoch=True)) ARGS.evt_list = [ARGS.start_time] # if not ARGS.evt_list and not ARGS.all_events: sys.stderr.write( "Error: Required argument missing. event_number|evt_list|all_events.\n" ) sys.exit(-1) # Event or shot line if ARGS.shot_line != None: if ARGS.shot_line == 0: ARGS.shot_line = "Event_t" else: ARGS.shot_line = "Event_t_{0:03d}".format(ARGS.shot_line) # elif not ARGS.start_time: sys.stderr.write("Error: Shot line or start time required.") sys.exit(-2) # Array or station line ARGS.station_array = "Array_t_{0:03d}".format(ARGS.station_array) # Order of channels in gather ARGS.channels = map(int, ARGS.channels.split(',')) # Stations to gather if ARGS.stations_to_gather: ARGS.stations_to_gather = map(int, ARGS.stations_to_gather.split(',')) ARGS.stations_to_gather.sort() ARGS.stations_to_gather = map(str, ARGS.stations_to_gather) if not os.path.exists(ARGS.out_dir): os.mkdir(ARGS.out_dir) os.chmod(ARGS.out_dir, 0777)