def qc_fields():
'''
Check each column for regular expression, type, range.
'''
n = 0
ret = []
for t in TABLE:
n += 1
for i in range(len(t)):
key = NAMES[i]
if key == 'Ignore':
continue
v = t[i]
if 'type' in COLS[key]:
ttype = COLS[key]['type']
else:
ttype = None
if 'help' in COLS[key]:
hhelp = COLS[key]['help']
else:
hhelp = None
if 're' in COLS[key]:
rre = COLS[key]['re']
else:
rre = None
if 'range' in COLS[key]:
rrange = COLS[key]['range']
else:
rrange = None
# Check regular expression
if not match_re(v, rre):
ret.append(
"{0}: Value of column {1} {2} does not match re. Help:"
" {3}".format(n, key, v, hhelp))
# Check type
if not match_type(v, ttype):
ret.append(
"{0}: Value of column {1} {2} does not match type. Type:"
" {3}".format(n, key, v, ttype))
# Check range
if not match_range(v, rrange):
ret.append(
"{0}: Value of column {1} {2} does not match expected"
" range. Range: {3}".format(n, key, v, rrange))
# Check if ascii time
if timeRE.match(key):
try:
timedoy.fdsn2epoch(v, fepoch=True)
except timedoy.TimeError:
try:
timedoy.passcal2epoch(v, fepoch=True)
except timedoy.TimeError:
ret.append(
"{0}: Value of column {1} {2} does not match"
" expected time string".format(n, key, v))
return ret
def _is_in(das, shot_time, length, si):
'''
Test to see if data is available for a given das starting at shot_time for length.
Check to see if there are any gaps greater than the sample interval.
Inputs:
das - The das serial number as a string
shot_time - The shot time as a timedoy object
length - Length is seconds as a float
si - Sample interval in seconds
Returns:
A tuple containing:
Match first sample time as a timedoy object, None if no match
Match last sample time as a timedoy object, None if no match
Gaps as a list of start and end times as timedoy objects
'''
if DATA.has_key(das): data = DATA[das]
else: data = []
shot_start_epoch = shot_time.epoch(fepoch=True)
shot_stop_epoch = shot_start_epoch + length
hits = []
gaps = []
for d in data:
data_start_epoch = d['window_start'].epoch(fepoch=True)
data_stop_epoch = d['window_stop'].epoch(fepoch=True)
# Call ph5api.is_in
if is_in(data_start_epoch, data_stop_epoch, shot_start_epoch,
shot_stop_epoch):
hits.append(d)
# Match no gaps
if len(hits) == 1:
return hits[0]['first_sample'], hits[0]['last_sample'], gaps
# No match
elif len(hits) == 0:
return None, None, gaps
# Match with gaps
else:
fs = None
ls = None
for h in hits:
if fs == None:
fs = h['first_sample']
if ls == None:
ls = h['last_sample']
delta = abs(
timedoy.passcal2epoch(h['last_sample']) -
timedoy.passcal2epoch(ls))
if delta > si:
gaps.append((ls, h['last_sample']))
ls = h['last_sample']
return fs, ls, gaps
def qc_time(t1, t2):
'''
Measure difference between deployment and pickup times.
'''
try:
e1 = timedoy.fdsn2epoch(t1)
except timedoy.TimeError:
e1 = timedoy.passcal2epoch(t1)
try:
e2 = timedoy.fdsn2epoch(t2)
except timedoy.TimeError:
e2 = timedoy.passcal2epoch(t2)
return e2 - e1
def get_times(key, value):
'''
Create time entries for Array_t_xxx or Event_t[_xxx]
'''
try:
fepoch = timedoy.fdsn2epoch(value, fepoch=True)
except timedoy.TimeError:
try:
fepoch = timedoy.passcal2epoch(value, fepoch=True)
except timedoy.TimeError:
# This SHOULD never happen
pre = key.split('/')[0]
LOGGER.error("Bad time value for {0} {1}.".format(key, value))
line = "\t{0}/ascii_s = {1}\n".format(pre, time.ctime(int(0)))
line += "\t{0}/epoch_l = {1}\n".format(pre, int(0))
line += "\t{0}/micro_seconds_i = {1}\n".format(
pre, int(0. * 1000000.))
line += "\t{0}/type_s = {1}\n".format(pre, 'BOTH')
return line
f, i = math.modf(fepoch)
pre = key.split('/')[0]
line = "\t{0}/ascii_s = {1}\n".format(pre, time.ctime(int(i)))
line += "\t{0}/epoch_l = {1}\n".format(pre, int(i))
line += "\t{0}/micro_seconds_i = {1}\n".format(pre, int(f * 1000000.))
line += "\t{0}/type_s = {1}\n".format(pre, 'BOTH')
return line
def doy_breakup(start_fepoch, length=86400):
"""
Given a start time epoch returns a next days equivalent epoch time and the
difference in seconds between the start and stop epoch times.
:param: start_fepoch
:type: float
:returns: stop_fepoch : next days stop epoch :type: float
seconds: difference in seconds between the start and end
epoch times :type: float
"""
passcal_start = epoch2passcal(float(start_fepoch))
start_passcal_list = passcal_start.split(":")
start_year = start_passcal_list[0]
start_doy = start_passcal_list[1]
start_hour = start_passcal_list[2]
start_minute = start_passcal_list[3]
start_second = start_passcal_list[4]
datestr = "{0}:{1}:{2}:{3}:{4}".format(start_year, start_doy, start_hour,
start_minute, start_second)
passcal_date = datetime.strptime(datestr, "%Y:%j:%H:%M:%S.%f")
next_passcal_date = passcal_date + timedelta(seconds=length)
next_passcal_date_str = next_passcal_date.strftime("%Y:%j:%H:%M:%S.%f")
stop_fepoch = passcal2epoch(next_passcal_date_str, fepoch=True)
seconds = stop_fepoch - start_fepoch
return stop_fepoch, seconds
def process_all():
'''
Process through each shot line, shot, array, station,
component (channel) and print matches to stdout
'''
Events = EVENT['Events']
for Event in Events:
shot_line = Event['shot_line']
shot_line_name = "Event_t_{0:03d}".format(int(shot_line))
for event in Event['Events']:
yr, doy, hr, mn, sc = event['time'].split(':')
shot_time = timedoy.TimeDOY(year=int(yr),
doy=int(doy),
hour=int(hr),
minute=int(mn),
second=float(sc))
if ARGS.offset_secs:
shot_time = shot_time + ARGS.offset_secs
shot_id = event['id']
Arrays = ARRAY['Arrays']
for Array in Arrays:
array_name = "Array_t_{0:03d}".format(int(Array['array']))
sample_rate = Array['sample_rate']
length = 65536. / sample_rate
cut_end = shot_time + length
for station in Array['Stations']:
chan = station['chan']
das = station['das']
station_id = station['id']
seed_id = station['seed_station_name']
fs, ls, gaps = _is_in(das, shot_time, length,
1. / sample_rate)
if fs is None:
fs = 'NA'
if ls is None:
ls = 'NA'
if ARGS.epoch:
if fs != 'NA':
fs = str(timedoy.passcal2epoch(fs, fepoch=True))
if ls != 'NA':
ls = str(timedoy.passcal2epoch(ls, fepoch=True))
line = [
shot_line_name, shot_id,
str(shot_time.epoch(fepoch=True)),
str(cut_end.epoch(fepoch=True)), array_name,
station_id, seed_id, das,
str(chan), fs, ls
]
else:
line = [
shot_line_name, shot_id,
shot_time.getPasscalTime(ms=True),
cut_end.getPasscalTime(ms=True), array_name,
station_id, seed_id, das,
str(chan), fs, ls
]
if ARGS.csv:
print ','.join(line)
else:
print ' '.join(line)
if len(gaps) != 0:
for g in gaps:
print "\t", g[0], g[1]
def create_cut(self, seed_network, ph5_station, seed_station,
station_cut_times, station_list, deployment, st_num,
array_code, experiment_id):
deploy = station_list[deployment][st_num]['deploy_time/epoch_l']
deploy_micro = station_list[deployment][
st_num]['deploy_time/micro_seconds_i']
pickup = station_list[deployment][st_num]['pickup_time/epoch_l']
pickup_micro = station_list[deployment][
st_num]['pickup_time/micro_seconds_i']
location = station_list[deployment][
st_num]['seed_location_code_s']
das = station_list[deployment][st_num]['das/serial_number_s']
das_manufacturer = station_list[deployment][st_num][
'das/manufacturer_s']
das_model = station_list[deployment][st_num][
'das/model_s']
sensor_type = " ".join([x for x in
[station_list[deployment][st_num][
'sensor/manufacturer_s'],
station_list[deployment][st_num][
'sensor/model_s']] if x])
receiver_n_i = station_list[deployment][st_num]['receiver_table_n_i']
response_n_i = station_list[deployment][st_num]['response_table_n_i']
if 'sample_rate_i' in station_list[deployment][0]:
sample_rate = station_list[deployment][st_num]['sample_rate_i']
sample_rate_multiplier = 1
if ('sample_rate_multiplier_i' in
station_list[deployment][st_num]):
sample_rate_multiplier = station_list[
deployment][st_num]['sample_rate_multiplier_i']
if self.sample_rate_list:
sample_list = self.sample_rate_list
if not ph5utils.does_pattern_exists(sample_list, sample_rate):
return
seed_channel, component = self.get_channel_and_component(
station_list, deployment, st_num)
if self.component:
component_list = self.component
if not ph5utils.does_pattern_exists(component_list, component):
return
if self.channel:
cha_patterns = self.channel
if not ph5utils.does_pattern_exists(cha_patterns, seed_channel):
return
if self.das_sn and self.das_sn != das:
return
if self.reqtype == "FDSN":
# trim user defined time range if it extends beyond the
# deploy/pickup times
if self.start_time:
if "T" not in self.start_time:
check_start_time = passcal2epoch(
self.start_time, fepoch=True)
if float(check_start_time) > float(deploy):
start_fepoch = self.start_time
sct = StationCutTime(
passcal2epoch(start_fepoch, fepoch=True)
)
station_cut_times.append(sct)
else:
sct = StationCutTime(deploy)
station_cut_times.append(sct)
else:
check_start_time = ph5utils.datestring_to_epoch(
self.start_time)
if float(check_start_time) > float(deploy):
sct = StationCutTime(
ph5utils.datestring_to_epoch(self.start_time))
station_cut_times.append(sct)
else:
sct = StationCutTime(deploy)
station_cut_times.append(sct)
if float(check_start_time) > float(pickup):
return
else:
sct = StationCutTime(
ph5api.fepoch(deploy, deploy_micro)
)
station_cut_times.append(sct)
for sct in station_cut_times:
start_fepoch = sct.time
if self.reqtype == "SHOT" or self.reqtype == "RECEIVER":
if self.offset:
# adjust starttime by an offset
start_fepoch += int(self.offset)
if self.length:
stop_fepoch = start_fepoch + self.length
else:
raise PH5toMSAPIError(
"Error - length is required for request by shot.")
elif self.reqtype == "FDSN":
if self.end_time:
if "T" not in self.end_time:
check_end_time = passcal2epoch(
self.end_time, fepoch=True)
if float(check_end_time) < float(pickup):
stop_fepoch = self.end_time
stop_fepoch = passcal2epoch(
stop_fepoch, fepoch=True)
else:
stop_fepoch = pickup
else:
check_end_time = ph5utils.datestring_to_epoch(
self.end_time)
if float(check_end_time) < float(pickup):
stop_fepoch = ph5utils.datestring_to_epoch(
self.end_time)
else:
stop_fepoch = pickup
if float(check_end_time) < float(deploy):
continue
elif self.length:
stop_fepoch = start_fepoch + self.length
else:
stop_fepoch = ph5api.fepoch(pickup, pickup_micro)
if (self.use_deploy_pickup is True and not
((int(start_fepoch) >= deploy and
int(stop_fepoch) <= pickup))):
# das not deployed within deploy/pickup time
continue
start_passcal = epoch2passcal(start_fepoch, sep=':')
start_passcal_list = start_passcal.split(":")
start_doy = start_passcal_list[1]
if self.doy_keep:
if start_doy not in self.doy:
continue
midnight_fepoch, secondLeftInday = \
ph5utils.inday_breakup(start_fepoch)
# if (stop_fepoch - start_fepoch) > 86400:
if (stop_fepoch - start_fepoch) > secondLeftInday:
seconds_covered = 0
total_seconds = stop_fepoch - start_fepoch
times_to_cut = []
if self.cut_len != 86400:
stop_time, seconds = ph5utils.doy_breakup(
start_fepoch, self.cut_len)
else:
stop_time, seconds = ph5utils.inday_breakup(start_fepoch)
seconds_covered = seconds_covered + seconds
times_to_cut.append([start_fepoch, stop_time])
start_time = stop_time
while seconds_covered < total_seconds:
if self.cut_len != 86400:
stop_time, seconds = ph5utils.doy_breakup(
start_time, self.cut_len)
else:
stop_time, seconds = ph5utils.inday_breakup(start_time)
seconds_covered += seconds
if stop_time > stop_fepoch:
times_to_cut.append([start_time, stop_fepoch])
break
times_to_cut.append([start_time, stop_time])
start_time = stop_time
else:
times_to_cut = [[start_fepoch, stop_fepoch]]
times_to_cut[-1][-1] = stop_fepoch
latitude = station_list[deployment][
st_num]['location/Y/value_d']
longitude = station_list[deployment][
st_num]['location/X/value_d']
elev = station_list[deployment][
st_num]['location/Z/value_d']
for starttime, endtime in tuple(times_to_cut):
try:
self.ph5.query_das_t(das,
component,
starttime,
endtime,
sample_rate,
sample_rate_multiplier
)
except experiment.HDF5InteractionError:
continue
station_x = StationCut(
seed_network,
experiment_id,
ph5_station,
seed_station,
array_code,
das,
das_manufacturer,
das_model,
sensor_type,
component,
seed_channel,
starttime,
endtime,
sample_rate,
sample_rate_multiplier,
self.notimecorrect,
location,
latitude,
longitude,
elev,
receiver_n_i,
response_n_i,
shot_id=sct.shot_id,
shot_lat=sct.shot_lat,
shot_lng=sct.shot_lng,
shot_elevation=sct.shot_elevation)
station_hash = hash(frozenset([seed_station, das, latitude,
longitude, sample_rate,
sample_rate_multiplier,
starttime, endtime]))
if station_hash in self.hash_list:
continue
else:
self.hash_list.append(station_hash)
yield station_x
def get_args():
''' Read command line argments '''
global ARGS, P5
import argparse
parser = argparse.ArgumentParser()
parser.usage = "Version: %s\n" % PROG_VERSION
parser.usage += "ph5toevt --eventnumber=shot --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n"
parser.usage += "\toptions:\n\t--array=array, --offset=seconds (float), --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n"
parser.usage += "ph5toevt --allevents --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n"
parser.usage += "\toptions:\n\t--array=array, --offset=seconds (float), --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n"
parser.usage += "ph5toevt --starttime=yyyy:jjj:hh:mm:ss[:.]sss --nickname=experiment_nickname --length=seconds [--path=ph5_directory_path] [options]\n"
parser.usage += "\toptions:\n\t--stoptime=yyyy:jjj:hh:mm:ss[:.]sss, --array=array, --reduction_velocity=km-per-second (float) --format=['SEGY']\n\n"
#parser.usage += "ph5toseg --all, --nickname=experiment_nickname [--path=ph5_directory_path] [--das=das_sn] [--station=station_id] [--doy=comma seperated doy list] [options]"
parser.usage += "\n\n\tgeneral options:\n\t--channel=[1,2,3]\n\t--sample_rate_keep=sample_rate\n\t--notimecorrect\n\t--decimation=[2,4,5,8,10,20]\n\t--out_dir=output_directory"
parser.description = "Generate SEG-Y gathers in shot order..."
# Usually master.ph5
parser.add_argument("-n",
"--nickname",
dest="ph5_file_prefix",
help="The ph5 file prefix (experiment nickname).",
metavar="ph5_file_prefix",
required=True)
# Path to the directory that holds master.ph5
parser.add_argument(
"-p",
"--path",
dest="ph5_path",
help="Path to ph5 files. Defaults to current directory.",
metavar="ph5_path",
default='.')
# SEED channel
parser.add_argument("--channel",
dest="seed_channel",
help="Filter on SEED channel.",
metavar="seed_channel")
# SEED network code
parser.add_argument("--network",
dest="seed_network",
help="Filter on SEED net code.",
metavar="seed_network")
# SEED loc code
parser.add_argument("--location",
dest="seed_location",
help="Filter on SEED loc code.",
metavar="seed_location")
# Channels. Will extract in order listed here. 'Usually' 1 -> Z, 2-> N, 3 -> E
parser.add_argument(
"-c",
"--channels",
action="store",
help="List of comma seperated channels to extract. Default = 1,2,3.",
type=str,
dest="channels",
metavar="channels",
default='1,2,3')
# Extract a single event
parser.add_argument("-e",
"--eventnumber",
action="store",
dest="event_number",
type=int,
metavar="event_number")
# Event id's in order, comma seperated
parser.add_argument(
"--event_list",
dest="evt_list",
help=
"Comma separated list of event id's to gather from defined or selected events.",
metavar="evt_list")
# Extract all events in Event_t
parser.add_argument("-E",
"--allevents",
action="store_true",
dest="all_events",
help="Extract all events in event table.",
default=False)
# The shot line number, 0 for Event_t
parser.add_argument("--shot_line",
dest="shot_line",
action="store",
help="The shot line number that holds the shots.",
type=int,
metavar="shot_line")
# External shot line file
parser.add_argument(
"--shot_file",
dest="shot_file",
action="store",
help=
"Input an external kef file that contains event information, Event_t.kef.",
type=str,
metavar="shot_file")
# Extract data for all stations starting at this time
parser.add_argument("-s",
"--starttime",
action="store",
dest="start_time",
type=str,
metavar="start_time")
# The array number
parser.add_argument("-A",
"--station_array",
dest="station_array",
action="store",
help="The array number that holds the station(s).",
type=int,
metavar="station_array",
required=True)
# Length of traces to put in gather
parser.add_argument("-l",
"--length",
action="store",
required=True,
type=int,
dest="length",
metavar="length")
# Start trace at time offset from shot time
parser.add_argument(
"-O",
"--seconds_offset_from_shot",
"--offset",
metavar="seconds_offset_from_shot",
help="Time in seconds from shot time to start the trace.",
type=float,
default=0.)
# Do not time correct texan data
parser.add_argument("-N",
"--notimecorrect",
action="store_false",
default=True,
dest="do_time_correct")
# Output directory
parser.add_argument("-o",
"--out_dir",
action="store",
dest="out_dir",
metavar="out_dir",
type=str,
default=".")
# Write to stdout
parser.add_argument("--stream",
action="store_true",
dest="write_stdout",
help="Write to stdout instead of a file.",
default=False)
# Use deploy and pickup times to determine where an instrument was deployed
parser.add_argument(
"--use_deploy_pickup",
action="store_true",
default=False,
help=
"Use deploy and pickup times to determine if data exists for a station.",
dest="deploy_pickup")
# Stations to gather, comma seperated
parser.add_argument(
"-S",
"--stations",
"--station_list",
dest="stations_to_gather",
help="Comma separated list of stations to receiver gather.",
metavar="stations_to_gather",
required=False)
# Filter out all sample rates except the listed
parser.add_argument("-r",
"--sample_rate_keep",
action="store",
dest="sample_rate",
metavar="sample_rate",
type=float)
# Apply a reduction velocity, km
parser.add_argument("-V",
"--reduction_velocity",
action="store",
dest="red_vel",
help="Reduction velocity in km/sec.",
metavar="red_vel",
type=float,
default="-1.")
# Decimate data. Decimation factor
parser.add_argument("-d",
"--decimation",
action="store",
choices=["2", "4", "5", "8", "10", "20"],
dest="decimation",
metavar="decimation")
# Convert geographic coordinated in ph5 to UTM before creating gather
parser.add_argument("-U",
"--UTM",
action="store_true",
dest="use_utm",
help="Fill SEG-Y headers with UTM instead of lat/lon.",
default=False)
# How to fill in the extended trace header
parser.add_argument("-x",
"--extended_header",
action="store",
dest="ext_header",
help="Extended trace header style: \
'P' -> PASSCAL, \
'S' -> SEG, \
'U' -> Menlo USGS, \
default = U",
choices=["P", "S", "U", "I", "N"],
default="U",
metavar="extended_header_style")
# Ignore channel in Das_t. Only useful with texans
parser.add_argument("--ic",
action="store_true",
dest="ignore_channel",
default=False)
# Allow traces to be 2^16 samples long vs 2^15
parser.add_argument("--break_standard",
action="store_false",
dest="break_standard",
help="Force traces to be no longer than 2^15 samples.",
default=True)
parser.add_argument("--debug",
dest="debug",
action="store_true",
default=False)
ARGS = parser.parse_args()
#print ARGS
try:
P5 = ph5api.PH5(path=ARGS.ph5_path, nickname=ARGS.ph5_file_prefix)
except Exception as e:
sys.stderr.write("Error: Can't open {0} at {1}.".format(
ARGS.ph5_file_prefix, ARGS.ph5_path))
sys.exit(-1)
#
if ARGS.shot_file:
if not ARGS.shot_line:
sys.stderr.write(
"Error: Shot line switch, --shot_line, required when using external shot file."
)
sys.exit(-3)
external = external_file.External(ARGS.shot_file)
ARGS.shot_file = external.Event_t
P5.Event_t_names = ARGS.shot_file.keys()
else:
P5.read_event_t_names()
#
if ARGS.event_number:
ARGS.evt_list = list([str(ARGS.event_number)])
elif ARGS.evt_list:
ARGS.evt_list = map(str, ARGS.evt_list.split(','))
elif ARGS.start_time:
ARGS.start_time = timedoy.TimeDOY(
epoch=timedoy.passcal2epoch(ARGS.start_time, fepoch=True))
ARGS.evt_list = [ARGS.start_time]
#
if not ARGS.evt_list and not ARGS.all_events:
sys.stderr.write(
"Error: Required argument missing. event_number|evt_list|all_events.\n"
)
sys.exit(-1)
# Event or shot line
if ARGS.shot_line != None:
if ARGS.shot_line == 0:
ARGS.shot_line = "Event_t"
else:
ARGS.shot_line = "Event_t_{0:03d}".format(ARGS.shot_line)
#
elif not ARGS.start_time:
sys.stderr.write("Error: Shot line or start time required.")
sys.exit(-2)
# Array or station line
ARGS.station_array = "Array_t_{0:03d}".format(ARGS.station_array)
# Order of channels in gather
ARGS.channels = map(int, ARGS.channels.split(','))
# Stations to gather
if ARGS.stations_to_gather:
ARGS.stations_to_gather = map(int, ARGS.stations_to_gather.split(','))
ARGS.stations_to_gather.sort()
ARGS.stations_to_gather = map(str, ARGS.stations_to_gather)
if not os.path.exists(ARGS.out_dir):
os.mkdir(ARGS.out_dir)
os.chmod(ARGS.out_dir, 0777)
