def main():
args = get_args(sys.argv[1:])
if args.nickname[-3:] == 'ph5':
ph5file = os.path.join(args.ph5path, args.nickname)
else:
ph5file = os.path.join(args.ph5path, args.nickname + '.ph5')
args.nickname += '.ph5'
PATH = os.path.dirname(args.ph5path) or '.'
# Debugging
os.chdir(PATH)
# Write log to file
ch = logging.FileHandler(os.path.join(".", "metadatatoph5.log"))
ch.setLevel(logging.INFO)
# Add formatter
formatter = logging.Formatter(LOGGING_FORMAT)
ch.setFormatter(formatter)
LOGGER.addHandler(ch)
if not os.path.exists(ph5file):
LOGGER.warning("{0} not found. Creating...".format(ph5file))
# Create ph5 file
ex = experiment.ExperimentGroup(nickname=ph5file,
currentpath=args.ph5path)
ex.ph5open(True) # Open ph5 file for editing
ex.initgroup()
# Update /Experiment_g/Receivers_g/Receiver_t
default_receiver_t = initialize_ph5.create_default_receiver_t()
initialize_ph5.set_receiver_t(default_receiver_t)
LOGGER.info(
"Removing temporary {0} kef file.".format(default_receiver_t))
os.remove(default_receiver_t)
ex.ph5close()
LOGGER.info("Done... Created new PH5 file {0}.".format(ph5file))
ph5_object = experiment.ExperimentGroup(nickname=args.nickname,
currentpath=args.ph5path)
ph5_object.ph5open(True)
ph5_object.initgroup()
metadata = MetadatatoPH5(ph5_object)
path, file_name = os.path.split(args.infile)
f = open(args.infile, "r")
inventory = metadata.read_metadata(f, file_name)
if inventory:
parsed_array = metadata.parse_inventory(inventory)
metadata.toph5(parsed_array)
ph5_object.ph5close()
def initialize_ph5(editmode=False):
''' Initialize the ph5 file '''
global EX, PATH, PH5
EX = experiment.ExperimentGroup(PATH, PH5)
EX.ph5open(editmode)
EX.initgroup()
def initializeExperiment():
global EX, PH5, PATH
EX = experiment.ExperimentGroup(currentpath=PATH, nickname=PH5)
EDIT = True
EX.ph5open(EDIT)
EX.initgroup()
def main():
global PH5, KEFFILE
get_args()
# Create ph5 file
EDITMODE = True
ex = experiment.ExperimentGroup(nickname=PH5)
ex.ph5open(EDITMODE)
ex.initgroup()
# Update Experiment_t
if KEFFILE:
k = kef.Kef(KEFFILE)
k.open()
k.read()
k.batch_update()
k.close()
if os.path.exists(RECEIVER_T):
k = kef.Kef(RECEIVER_T)
k.open()
k.read()
k.batch_update()
k.close()
else:
sys.stderr.write(
"Warning: /Experiment_g/Receivers_g/Receiver_t not set!\n")
os.remove("kef.tmp")
# Close PH5 file
ex.ph5close()
print "Done..."
os.remove("kef.tmp")
def openPH5 (filename) :
#filename = os.path.join (PATH, filename)
#sys.stderr.write ("*** Opening: {0} ".format (filename))
exrec = experiment.ExperimentGroup (nickname = filename)
exrec.ph5open (True)
exrec.initgroup ()
return exrec
def setUp(self):
self.path = 'ph5/test_data/miniseedph5'
try:
os.mkdir(self.path)
except OSError:
print ("Creation of the directory %s failed" % self.path)
else:
print ("Successfully created the directory %s " % self.path)
ph5_object = experiment.ExperimentGroup(
nickname='master.ph5',
currentpath=self.path)
ph5_object.ph5open(True)
ph5_object.initgroup()
default_receiver_t = initialize_ph5.create_default_receiver_t()
initialize_ph5.set_receiver_t(default_receiver_t)
os.remove(default_receiver_t)
ph5_object.initgroup()
self.obs = obspytoph5.ObspytoPH5(
ph5_object,
self.path,
1,
1)
self.obs.verbose = True
ph5_object.ph5flush()
ph5_object.ph5_g_sorts.update_local_table_nodes()
def main():
try:
args = get_args()
except ValueError:
pass # Error parsing arguments
else:
# Create ph5 file
ex = experiment.ExperimentGroup(nickname=args.outfile)
ex.ph5open(True) # Open ph5 file for editing
ex.initgroup()
# Update /Experiment_g/Experiment_t
set_experiment_t(args.experiment_t)
# Update /Experiment_g/Receivers_g/Receiver_t
if args.receiver_t:
set_receiver_t(args.receiver_t)
else:
LOGGER.warning("Experiment_g/Receivers_g/Receiver_t set using "
"default values. Use --receiver_t option to supply "
"a Receiver_t kef file.")
default_receiver_t = create_default_receiver_t()
set_receiver_t(default_receiver_t)
LOGGER.info(
"Removing temporary {0} kef file.".format(default_receiver_t))
os.remove(default_receiver_t)
# Close PH5 file
ex.ph5close()
LOGGER.info("Done... Created new PH5 file {0}.".format(args.outfile))
def reopen(ex):
filename = ex.filename
ex.ph5close()
ex = experiment.ExperimentGroup(nickname=filename)
ex.ph5open(True)
ex.initgroup()
return ex
def initializeExperiment():
global EX
EX = experiment.ExperimentGroup(nickname=PH5)
EDIT = True
EX.ph5open(EDIT)
EX.initgroup()
def initializeExperiment():
''' Open PH5 file, master.ph5 '''
global EX, PH5
EX = experiment.ExperimentGroup(nickname=PH5)
EDIT = True
EX.ph5open(EDIT)
EX.initgroup()
def initializeExperiment (nickname) :
global EX, PH5
#reload (tables)
EX = experiment.ExperimentGroup (nickname = PH5)
EDIT = True
EX.ph5open (EDIT)
EX.initgroup ()
def initialize_ph5(editmode=False):
''' Initialize the ph5 file '''
global EX, PATH, PH5
try:
EX = experiment.ExperimentGroup(PATH, PH5)
EX.ph5open(editmode)
EX.initgroup()
except Exception:
LOGGER.error("Cannot open PH5 file. Use -h argument for help.")
sys.exit()
def openPH5(filename):
''' Open PH5 file, miniPH5_xxxxx.ph5 '''
try:
if EXREC.ph5.isopen:
if EXREC.filename != filename:
EXREC.ph5close()
else:
return EXREC
except BaseException:
pass
exrec = experiment.ExperimentGroup(nickname=filename)
exrec.ph5open(True)
exrec.initgroup()
return exrec
def openPH5(filename):
''' Open PH5 file, miniPH5_xxxxx.ph5 '''
try:
if EXREC.ph5.isopen:
if EXREC.filename != filename:
EXREC.ph5close()
else:
return EXREC
except BaseException:
pass
# sys.stderr.write ("*** Opening: {0} ".format (filename))
exrec = experiment.ExperimentGroup(nickname=filename)
exrec.ph5open(True)
exrec.initgroup()
return exrec
def openmini(self, mini_num):
"""
Open PH5 file, miniPH5_xxxxx.ph5
:type: str
:param mini_num: name of mini file to open
:return class: ph5.core.experiment, str: name
"""
mini_num = str(mini_num).zfill(5)
filename = "miniPH5_{0}.ph5".format(mini_num)
exrec = experiment.ExperimentGroup(nickname=filename,
currentpath=self.ph5_path)
exrec.ph5open(True)
exrec.initgroup()
return exrec, filename
def setUp(self):
self.path = 'ph5/test_data/miniseedph5'
try:
os.mkdir(self.path)
except OSError:
print("Creation of the directory %s failed" % self.path)
else:
print("Successfully created the directory %s " % self.path)
self.ph5_object = experiment.ExperimentGroup(nickname='master.ph5',
currentpath=self.path)
self.ph5_object.ph5open(True)
self.ph5_object.initgroup()
default_receiver_t = initialize_ph5.create_default_receiver_t()
initialize_ph5.set_receiver_t(default_receiver_t)
os.remove(default_receiver_t)
self.metadata = metadatatoph5.MetadatatoPH5(self.ph5_object)
def reformat_das_t(ph5object, das_sn, ph5, path):
'''
remove das_t and reformat from ph3 to pn4
:param ph5object: ph5 object where the das table will be deleted
:param das_sn: serial number of das to be deleted Ex: '1X1111'
Ex: 'Das_t_1X1111'
:param ph5: name of ph5 file (str)
:param path: path to ph5 file (str)
:return
backupfile: name of the kef file to backup the deleted table (str)
datapath: path to the table in ph5 structure
ph5object: ph5 object of which das table has been deleted
'''
# get mini_file that keep the passed das's data
index_rows, keys = ph5object.ph5_g_maps.read_index()
for i in index_rows:
if i['serial_number_s'] == das_sn:
mini_filename = i['external_file_name_s']
break
# open mini ph5 file to reformat das_t from pn3 to pn4
# because das_t is read-only when opened from ph5object
exrec = experiment.ExperimentGroup(nickname=mini_filename,
currentpath=path)
exrec.ph5open(True)
exrec.initgroup()
# remove das_t and re-initialize das_t with pn4's structure
das_t = exrec.ph5_g_receivers.ph5.get_node(
'/Experiment_g/Receivers_g/Das_g_%s' % das_sn,
name='Das_t',
classname='Table')
das_t.remove()
experiment.initialize_table(exrec.ph5_g_receivers.ph5,
'/Experiment_g/Receivers_g/Das_g_%s' % das_sn,
'Das_t',
columns.Data,
expectedrows=1000)
exrec.ph5close()
# The changes have happened on exrec, NOT on ph5object.
# Now need to close and re-open ph5object to update all those changes.
ph5object.close()
ph5object = ph5api.PH5(path=path, nickname=ph5, editmode=True)
# ph5object has been reopened, need to return for uses afterward
return ph5object
def test_main(self):
"""
test main function
"""
testargs = [
'metadatatoph5', '-n', 'master.ph5', '-f',
'ph5/test_data/metadata/station.xml'
]
with patch.object(sys, 'argv', testargs):
metadatatoph5.main()
self.assertTrue(os.path.isfile('master.ph5'))
ph5_object = experiment.ExperimentGroup(nickname='master.ph5')
ph5_object.ph5open(True)
ph5_object.initgroup()
array_names = ph5_object.ph5_g_sorts.names()
self.assertEqual(['Array_t_001', 'Array_t_002', 'Array_t_003'],
array_names)
ret, keys = ph5_object.ph5_g_sorts.read_arrays('Array_t_001')
key = [
'id_s', 'location/X/value_d', 'location/X/units_s',
'location/Y/value_d', 'location/Y/units_s', 'location/Z/value_d',
'location/Z/units_s', 'location/coordinate_system_s',
'location/projection_s', 'location/ellipsoid_s',
'location/description_s', 'deploy_time/ascii_s',
'deploy_time/epoch_l', 'deploy_time/micro_seconds_i',
'deploy_time/type_s', 'pickup_time/ascii_s', 'pickup_time/epoch_l',
'pickup_time/micro_seconds_i', 'pickup_time/type_s',
'das/serial_number_s', 'das/model_s', 'das/manufacturer_s',
'das/notes_s', 'sensor/serial_number_s', 'sensor/model_s',
'sensor/manufacturer_s', 'sensor/notes_s', 'description_s',
'seed_band_code_s', 'sample_rate_i', 'sample_rate_multiplier_i',
'seed_instrument_code_s', 'seed_orientation_code_s',
'seed_location_code_s', 'seed_station_name_s', 'channel_number_i',
'receiver_table_n_i', 'response_table_n_i'
]
self.assertEqual(key, keys)
self.assertEqual(1, len(ret))
self.assertEqual('5553', ret[0]['das/serial_number_s'])
self.assertEqual('H', ret[0]['seed_instrument_code_s'])
self.assertEqual('H', ret[0]['seed_band_code_s'])
self.assertEqual('N', ret[0]['seed_orientation_code_s'])
ph5_object.ph5close()
os.remove('master.ph5')
os.remove('metadatatoph5.log')
def mini_map(self, existing_minis):
"""
:type list
:param existing_minis: A list of mini_files with path
:return: list of tuples containing
what mini file contains what serial #s
"""
mini_map = list()
for mini in existing_minis:
mini_num = int(mini.split('.')[-2].split('_')[-1])
exrec = experiment.ExperimentGroup(nickname=mini)
exrec.ph5open(True)
exrec.initgroup()
all_das = exrec.ph5_g_receivers.alldas_g()
das_list = list()
for g in all_das:
name = g.split('_')[-1]
das_list.append(name)
mini_map.append((mini_num, das_list))
exrec.ph5close()
return mini_map
def get_args():
''' Parse input args
-r raw file
-f file containing list of raw files
-o output file
-k kef file # REMOVED
-d dep file # REMOVED
-w windows file
-p par_file
-P print out table list
-v be verbose
-M create a specific number of miniPH5 files
-S First index of miniPH5_xxxxx.ph5
'''
global FILES, PH5, WINDOWS, PARAMETERS, SR, NUM_MINI, VERBOSE, DEBUG
global FIRST_MINI
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.usage = ("1302ph5 [--help][--raw raw_file | --file file_list_file]"
" --nickname output_file_prefix")
parser.description = (
"Read a raw rt-130 files into ph5 format. v{0}".format(PROG_VERSION))
parser.add_argument("-r",
"--raw",
dest="rawfile",
help="RT-130 raw file",
metavar="raw_file")
parser.add_argument("-f",
"--file",
dest="infile",
help="File containing list of RT-130 raw file names.",
metavar="file_list_file")
parser.add_argument("-n",
"--nickname",
dest="outfile",
help="The ph5 file prefix (experiment nick name).",
metavar="output_file_prefix")
parser.add_argument("-M",
"--num_mini",
dest="num_mini",
help="Create a given number miniPH5_xxxxx.ph5 files.",
metavar="num_mini",
type=int,
default=None)
parser.add_argument("-S",
"--first_mini",
dest="first_mini",
help="The index of the first miniPH5_xxxxx.ph5 file.",
metavar="first_mini",
type=int,
default=1)
parser.add_argument("-s",
"--samplerate",
dest="samplerate",
help="Extract only data at given sample rate.",
metavar="samplerate")
parser.add_argument("-w",
"--windows_file",
dest="windows_file",
help=("File containing list of time windows to "
"process. \n"
"Window start time Window length, seconds\n"
"----------------- ----\n"
"YYYY:JJJ:HH:MM:SS SSSS"),
metavar="windows_file")
parser.add_argument("-p",
"--parfile",
dest="par_file",
help=("[Used to set sample rate and gain in the case "
"of a missing event header.]\n"
"Parameter file used to set samplerate, gain.\n"
"The file contains colon separated lists.\n"
"The first line describes the order and the "
"first char must be '#'.\n"
"As example the first four lines could be:\n\n"
"9882; 1; 1; 40; x1\n"
"9882; 2; 1; 40; x1\n"
"9882; 2; 1; 40; x1\n"
"9882; 1; 2; 1; x32\n"
"9882; 2; 2; 1; x32\n"
"9882; 3; 2; 1; x32\n\n"
"Allowed fields: das;station;refchan;refstrm;"
"samplerate;gain"),
metavar="par_file")
parser.add_argument("-P",
help="Do print",
dest="doprint",
action="store_true",
default=False)
parser.add_argument("-v",
help="Verbose logging",
dest="verbose",
action="store_true",
default=False)
parser.add_argument("-d",
help="Debug",
dest="debug",
action="store_true",
default=False)
args = parser.parse_args()
FILES = []
PH5 = None
SR = args.samplerate
NUM_MINI = args.num_mini
FIRST_MINI = args.first_mini
VERBOSE = args.verbose
DEBUG = args.debug
if args.debug:
VERBOSE = 2
if args.infile is not None:
read_infile(args.infile)
elif args.rawfile is not None:
FILES.append(args.rawfile)
if args.outfile is not None:
PH5 = args.outfile
if args.windows_file is not None:
WINDOWS = read_windows_file(args.windows_file)
else:
WINDOWS = None
if args.doprint is not False:
ex = experiment.ExperimentGroup()
ex.ph5open(True)
ex.initgroup()
ex.ph5close()
sys.exit()
if args.par_file is not None:
if not read_par_file(args.par_file):
LOGGER.error("Failed to read: {0}".format(args.par_file))
sys.exit()
else:
PARAMETERS = {}
if PH5 is None:
LOGGER.error("Missing required option. Try --help")
sys.exit()
if not os.path.exists(PH5) and not os.path.exists(PH5 + '.ph5'):
LOGGER.error("{0} does not exist!".format(PH5))
sys.exit()
else:
# Set up logging
# Write log to file
ch = logging.FileHandler(os.path.join('.', "1302ph5.log"))
ch.setLevel(logging.INFO)
# Add formatter
formatter = logging.Formatter(LOGGING_FORMAT)
ch.setFormatter(formatter)
LOGGER.addHandler(ch)
def initialize_ex(nickname, path, editmode=False):
ex = experiment.ExperimentGroup(nickname=nickname, currentpath=path)
ex.ph5open(editmode)
ex.initgroup()
return ex
def get_args () :
''' Parse input args
-r raw file
-f file containing list of raw files
-o output file
-k kef file # REMOVED
-d dep file # REMOVED
-w windows file
-p par_file
-P print out table list
-v be verbose
-M create a specific number of miniPH5 files
-S First index of miniPH5_xxxxx.ph5
'''
global FILES, PH5, WINDOWS, PARAMETERS, SR, NUM_MINI, VERBOSE, DEBUG, FIRST_MINI
from optparse import OptionParser
oparser = OptionParser ()
oparser.usage = "1302ph5 [--help][--raw raw_file | --file file_list_file] --nickname output_file_prefix"
oparser.description = "Read a raw rt-130 files into ph5 format. v%s" % PROG_VERSION
oparser.add_option ("-r", "--raw", dest = "rawfile",
help="RT-130 raw file", metavar="raw_file")
oparser.add_option ("-f", "--file", dest = "infile",
help = "File containing list of RT-130 raw file names.",
metavar = "file_list_file")
oparser.add_option ("-n", "--nickname", dest = "outfile",
help="The ph5 file prefix (experiment nick name).",
metavar = "output_file_prefix")
#oparser.add_option ("-k", "--kef", dest = "keffile",
#help = "Kitchen Exchange Format file.",
#metavar = "kef_file")
#oparser.add_option ("-d", "--dep", dest = "depfile",
#help = "Rawmeet dep file.",
#metavar = "dep_file")
oparser.add_option ("-M", "--num_mini", dest = "num_mini",
help = "Create a given number of miniPH5_xxxxx.ph5 files.",
metavar = "num_mini", type = 'int', default = None)
oparser.add_option ("-S", "--first_mini", dest = "first_mini",
help = "The index of the first miniPH5_xxxxx.ph5 file.",
metavar = "first_mini", type = 'int', default = 1)
oparser.add_option ("-s", "--samplerate", dest = "samplerate",
help = "Extract only data at given sample rate.",
metavar = "samplerate")
oparser.add_option ("-w", "--windows_file", dest = "windows_file",
help = "File containing list of time windows to process.\n\
Window start time Window length, seconds\n\
----------------- ----\n\
YYYY:JJJ:HH:MM:SS SSSS",
metavar = "windows_file")
oparser.add_option ("-p", "--parfile", dest = "par_file",
help = "[Used to set sample rate and gain in the case of a missing event header.]\n\
Parameter file used to set samplerate, gain.\n\
The file contains colon separated lists.\n\
The first line describes the\n\
order and the first char must be '#'.\n\
As example the first four lines could be:\n\
#das;refchan;refstrm;samplerate;gain\n\
9882; 1; 1; 40; x1\n\
\n\
9882; 2; 1; 40; x1\n\
\n\
9882; 3; 1; 40; x1\n\
\n\
9882; 1; 2; 1; x32\n\
\n\
9882; 2; 2; 1; x32\n\
\n\
9882; 3; 2; 1; x32\n\
\n\
Allowed fields: das;station;refchan;refstrm;samplerate;gain",
metavar = "par_file")
oparser.add_option ("-P", dest = "doprint", action = "store_true", default = False)
oparser.add_option ("-v", dest="verbose", action="store_true", default=False)
oparser.add_option ("-d", dest="debug", action="store_true", default=False)
options, args = oparser.parse_args()
#print options.outfile
FILES = []
PH5 = None
#KEFFILE = None
#DEPFILE = None
SR = options.samplerate
NUM_MINI = options.num_mini
FIRST_MINI = options.first_mini
VERBOSE = options.verbose
DEBUG = options.debug
if options.debug :
VERBOSE = 2
if options.infile != None :
read_infile (options.infile)
elif options.rawfile != None :
FILES.append (options.rawfile)
if options.outfile != None :
PH5 = options.outfile
if options.windows_file != None :
WINDOWS = read_windows_file (options.windows_file)
else :
WINDOWS = None
if options.doprint != False :
ex = experiment.ExperimentGroup ()
ex.ph5open (True)
ex.initgroup ()
keys (ex)
ex.ph5close ()
sys.exit ()
#if options.keffile != None :
#KEFFILE = options.keffile
#if options.depfile != None :
#DEPFILE = options.depfile
if options.par_file != None :
if not read_par_file (options.par_file) :
sys.stderr.write ("Error: Failed to read: %s\n" % options.par_file)
sys.exit ()
else :
PARAMETERS = {}
if PH5 == None :
#print H5, FILES
sys.stderr.write ("Error: Missing required option. Try --help\n")
sys.exit ()
if not os.path.exists (PH5) and not os.path.exists (PH5 + '.ph5') :
sys.stderr.write ("Error: %s does not exist!\n" % PH5)
sys.exit ()
else :
# Set up logging
#if not os.path.exists (OUTPATH) :
#os.mkdir (OUTPATH)
logging.basicConfig (
filename = os.path.join ('.', "1302ph5.log"),
format = "%(asctime)s %(message)s",
level = logging.INFO)
def openPH5(filename):
LOGGER.info("Opening: {0}".format(filename))
exrec = experiment.ExperimentGroup(nickname=filename)
exrec.ph5open(True)
exrec.initgroup()
return exrec
def load_response(self, path, nickname, data, input_csv):
ph5 = tables.open_file(os.path.join(path, nickname), "a")
with open(input_csv, "r") as f:
csv = f.readlines()
f.close()
loaded_das = []
loaded_sensor = []
for line in csv:
line_list = line.split(",")
if line_list[0] == 'Das Model':
continue
else:
if line_list[5] != "":
name = str(
line_list[0] +
"_" +
line_list[2] +
"_" +
line_list[3] +
"_" +
line_list[4])
if name not in loaded_das:
with open(line_list[5].rstrip(), "r") as f:
das_data = f.readlines()
f.close()
try:
name = name.replace(
" ", "")
name = name.translate(None, ',-=.')
ph5.create_array(
ph5.root.Experiment_g.Responses_g,
name.replace(
" ", ""), das_data)
loaded_das.append(name)
LOGGER.info(
"Loaded {0}".format(name.replace(" ", "")))
except BaseException:
LOGGER.warning(
"Could not load {0}"
.format(name.replace(" ", "")))
if len(line_list) >= 6:
if line_list[6] == '\n':
continue
if line_list[1] not in loaded_sensor:
with open(line_list[6].rstrip(), "r") as f:
sensor_data = f.readlines()
f.close()
try:
name = line_list[1].replace(
" ", "")
name = name.translate(None, ',-=.')
ph5.create_array(
ph5.root.Experiment_g.Responses_g,
name.replace(
" ", ""), sensor_data)
loaded_sensor.append(line_list[1])
LOGGER.info(
"Loaded {0}"
.format(name.replace(" ", "")))
except BaseException:
LOGGER.warning(
"Could not load {0}"
.format(name.replace(" ", "")))
ph5.close()
ph5_object = experiment.ExperimentGroup(nickname=nickname,
currentpath=path)
ph5_object.ph5open(True)
ph5_object.initgroup()
ret, blah = ph5_object.ph5_g_responses.read_responses()
data_list = []
data_update = []
for station in data:
data_list.append([str(station.das_model),
str(station.sensor_model),
str(station.sample_rate),
str(station.sample_rate_multiplier),
str(station.gain),
str(station.bit_weight),
str(station.bit_weight_units),
str(station.gain_units)])
unique_list = [list(x) for x in set(tuple(x) for x in data_list)]
# get highest n_i
n_i = 0
for entry in ret:
if entry['n_i'] >= n_i:
n_i = entry['n_i']
n_i = n_i + 1
no_resp = list()
for entry in ret:
if (not entry['response_file_das_a']
and not entry['response_file_sensor_a']):
no_resp.append(entry)
has_resp = list()
for entry in ret:
if entry['response_file_das_a'] or entry['response_file_sensor_a']:
has_resp.append(entry)
final_ret = list()
final_ret.extend(has_resp)
for x in unique_list:
response_entry = {}
if x[0]:
name = str(x[0]) + "_" + str(x[2]) + "_" + \
str(x[3]) + "_" + str(x[4])
name_full = '/Experiment_g/' +\
'Responses_g/' + name.replace(" ", "")
name_full = name_full.translate(None, ',-=.')
response_entry['response_file_das_a'] = name_full
if x[1]:
sens = x[1]
sens = sens.translate(None, ',-=.')
name = '/Experiment_g/' +\
'Responses_g/' + sens
response_entry['response_file_sensor_a'] = name
response_entry['bit_weight/value_d'] = x[5]
response_entry['bit_weight/units_s'] = x[6]
response_entry['gain/value_i'] = x[4]
response_entry['gain/units_s'] = x[7]
for resp in no_resp:
if (str(resp['bit_weight/value_d']) ==
str(response_entry['bit_weight/value_d']) and
str(resp['gain/value_i']) ==
str(response_entry['gain/value_i'])):
response_entry['n_i'] = resp['n_i']
final_ret.append(response_entry)
data_update.append([x[0], x[1], x[2], x[3], x[4], n_i])
ph5_object.ph5_g_responses.nuke_response_t()
final_ret.sort()
for entry in final_ret:
ref = columns.TABLES['/Experiment_g/Responses_g/Response_t']
columns.populate(ref, entry, None)
ph5_object.ph5close()
LOGGER.info(
"response_t.kef written into PH5")
for station in data:
for x in data_update:
if station.das_model == x[0] and str(
station.sensor_model) == x[1] and int(
station.sample_rate) == int(
x[2]) and int(
station.sample_rate_multiplier) ==\
int(x[3]) and int(station.gain) == int(x[4]):
station.response_n_i = x[5]
true_sr =\
float(station.sample_rate) /\
float(station.sample_rate_multiplier)
if true_sr < 1.0:
station.response_n_i = None
return data
def test_main(self):
"""
test main
"""
# first need to run metadatatoph5
testargs = ['metadatatoph5', '-n', 'master.ph5', '-f',
'ph5/test_data/metadata/station.xml']
with patch.object(sys, 'argv', testargs):
metadatatoph5.main()
# first need to run obspytoph5
testargs = ['obspytoph5', '-n', 'master.ph5', '-d',
'ph5/test_data/miniseed/']
with patch.object(sys, 'argv', testargs):
obspytoph5.main()
self.assertTrue(os.path.isfile('master.ph5'))
self.assertTrue(os.path.isfile('miniPH5_00001.ph5'))
ph5_object = experiment.ExperimentGroup(
nickname='master.ph5')
ph5_object.ph5open(True)
ph5_object.initgroup()
node = ph5_object.ph5_g_receivers.getdas_g('5553')
ph5_object.ph5_g_receivers.setcurrent(node)
ret, das_keys = ph5_object.ph5_g_receivers.read_das()
keys = ['array_name_SOH_a', 'array_name_data_a', 'array_name_event_a',
'array_name_log_a', 'channel_number_i', 'event_number_i',
'raw_file_name_s', 'receiver_table_n_i', 'response_table_n_i',
'sample_count_i', 'sample_rate_i', 'sample_rate_multiplier_i',
'stream_number_i', 'time/ascii_s', 'time/epoch_l',
'time/micro_seconds_i', 'time/type_s', 'time_table_n_i']
self.assertEqual(keys, das_keys)
self.assertEqual(
'ph5/test_data/miniseed/0407HHN.m',
ret[0]['raw_file_name_s'])
self.assertEqual(
'ph5/test_data/miniseed/0407LHN.m',
ret[1]['raw_file_name_s'])
ph5_object.ph5close()
os.remove('master.ph5')
os.remove('miniPH5_00001.ph5')
os.remove('metadatatoph5.log')
os.remove('datatoph5.log')
# first need to run metadatatoph5
testargs = ['metadatatoph5', '-n', 'master.ph5', '-f',
'ph5/test_data/metadata/station.xml']
with patch.object(sys, 'argv', testargs):
metadatatoph5.main()
# first need to run obspytoph5
testargs = ['obspytoph5', '-n', 'master.ph5', '-f',
'ph5/test_data/miniseed/0407HHN.ms']
with patch.object(sys, 'argv', testargs):
obspytoph5.main()
self.assertTrue(os.path.isfile('master.ph5'))
self.assertTrue(os.path.isfile('miniPH5_00001.ph5'))
ph5_object = experiment.ExperimentGroup(
nickname='master.ph5')
ph5_object.ph5open(True)
ph5_object.initgroup()
node = ph5_object.ph5_g_receivers.getdas_g('5553')
ph5_object.ph5_g_receivers.setcurrent(node)
ret, das_keys = ph5_object.ph5_g_receivers.read_das()
keys = ['array_name_SOH_a', 'array_name_data_a', 'array_name_event_a',
'array_name_log_a', 'channel_number_i', 'event_number_i',
'raw_file_name_s', 'receiver_table_n_i', 'response_table_n_i',
'sample_count_i', 'sample_rate_i', 'sample_rate_multiplier_i',
'stream_number_i', 'time/ascii_s', 'time/epoch_l',
'time/micro_seconds_i', 'time/type_s', 'time_table_n_i']
self.assertEqual(keys, das_keys)
self.assertEqual(
'ph5/test_data/miniseed/0407HHN.m',
ret[0]['raw_file_name_s'])
ph5_object.ph5close()
os.remove('master.ph5')
os.remove('miniPH5_00001.ph5')
os.remove('metadatatoph5.log')
os.remove('datatoph5.log')
# first need to run metadatatoph5
testargs = ['metadatatoph5', '-n', 'master.ph5', '-f',
'ph5/test_data/metadata/station.xml']
with patch.object(sys, 'argv', testargs):
metadatatoph5.main()
# now make a list for obspytoph5
f = open("test_list", "w")
f.write("ph5/test_data/miniseed/0407HHN.ms")
f.close()
# first need to run obspytoph5
testargs = ['obspytoph5', '-n', 'master.ph5', '-l',
'test_list']
with patch.object(sys, 'argv', testargs):
obspytoph5.main()
self.assertTrue(os.path.isfile('master.ph5'))
self.assertTrue(os.path.isfile('miniPH5_00001.ph5'))
ph5_object = experiment.ExperimentGroup(
nickname='master.ph5')
ph5_object.ph5open(True)
ph5_object.initgroup()
node = ph5_object.ph5_g_receivers.getdas_g('5553')
ph5_object.ph5_g_receivers.setcurrent(node)
ret, das_keys = ph5_object.ph5_g_receivers.read_das()
keys = ['array_name_SOH_a', 'array_name_data_a', 'array_name_event_a',
'array_name_log_a', 'channel_number_i', 'event_number_i',
'raw_file_name_s', 'receiver_table_n_i', 'response_table_n_i',
'sample_count_i', 'sample_rate_i', 'sample_rate_multiplier_i',
'stream_number_i', 'time/ascii_s', 'time/epoch_l',
'time/micro_seconds_i', 'time/type_s', 'time_table_n_i']
self.assertEqual(keys, das_keys)
self.assertEqual(
'ph5/test_data/miniseed/0407HHN.m',
ret[0]['raw_file_name_s'])
ph5_object.ph5close()
os.remove('master.ph5')
os.remove('miniPH5_00001.ph5')
os.remove('metadatatoph5.log')
os.remove('datatoph5.log')
os.remove('test_list')
# =============================================================================
# Test
# =============================================================================
#ts_fn = r"c:\Users\jpeacock\Documents\GitHub\sandbox\ts_test.EX"
ph5_fn = r"c:\Users\jpeacock\Documents\test_ph5.ph5"
#nfn = r"c:\Users\jpeacock\OneDrive - DOI\MountainPass\FieldWork\LP_Data\Mnp300a\DATA.BIN"
fn_list = glob.glob(r"c:\Users\jpeacock\Documents\imush\O015\*.Z3D")
st = datetime.datetime.now()
if os.path.exists(ph5_fn):
try:
os.remove(ph5_fn)
except PermissionError:
ph5_obj = experiment.ExperimentGroup(
nickname='test_ph5', currentpath=os.path.dirname(ph5_fn))
ph5_obj.ph5open(True)
ph5_obj.ph5close()
### initialize a PH5 object
ph5_obj = experiment.ExperimentGroup(nickname='test_ph5',
currentpath=os.path.dirname(ph5_fn))
ph5_obj.ph5open(True)
ph5_obj.initgroup()
### initialize mt2ph5 object
mt_obj = mttoph5.MTtoPH5()
mt_obj.ph5_obj = ph5_obj
# we give it a our trace and should get a message
# back saying done as well as an index table to be loaded
def main():
args = get_args(sys.argv[1:])
if args.nickname[-3:] == 'ph5':
ph5file = os.path.join(args.ph5path, args.nickname)
else:
ph5file = os.path.join(args.ph5path, args.nickname + '.ph5')
args.nickname += '.ph5'
PATH = os.path.dirname(args.ph5path) or '.'
# Debugging
os.chdir(PATH)
# Write log to file
ch = logging.FileHandler(os.path.join(".", "datatoph5.log"))
ch.setLevel(logging.INFO)
# Add formatter
formatter = logging.Formatter(LOGGING_FORMAT)
ch.setFormatter(formatter)
LOGGER.addHandler(ch)
if not os.path.exists(ph5file):
LOGGER.warning("{0} not found. Creating...".format(ph5file))
# Create ph5 file
ex = experiment.ExperimentGroup(nickname=ph5file)
ex.ph5open(True) # Open ph5 file for editing
ex.initgroup()
# Update /Experiment_g/Receivers_g/Receiver_t
default_receiver_t = initialize_ph5.create_default_receiver_t()
initialize_ph5.set_receiver_t(default_receiver_t)
LOGGER.info(
"Removing temporary {0} kef file.".format(default_receiver_t))
os.remove(default_receiver_t)
ex.ph5close()
LOGGER.info("Done... Created new PH5 file {0}.".format(ph5file))
# Produce list of valid files from file, list, and dirs
valid_files = list()
if args.rawfile:
if _is_mseed(args.rawfile):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to process list.".format(args.rawfile))
valid_files.append((args.rawfile, 'MSEED'))
else:
LOGGER.info("{0} is a NOT valid miniSEED file.".format(
args.rawfile))
try:
if _is_mseed(args.infile):
LOGGER.error("The given list file is a miniseed file. You have "
"been confused between two option -r and -f.")
sys.exit()
except TypeError:
pass
if args.infile:
LOGGER.info("Checking list...")
with open(args.infile) as f:
content = f.readlines()
for line in content:
if os.path.isfile(line.rstrip()):
if _is_mseed(line.rstrip()):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to"
"process list.".format(line.rstrip()))
valid_files.append((line.rstrip(), 'MSEED'))
else:
LOGGER.info("{0} is NOT a valid miniSEED "
"file.".format(line.rstrip()))
else:
LOGGER.info("{0} does not exist".format(line.rstrip()))
if args.indir:
LOGGER.info("Scanning directory {0} and sub directories...".format(
args.indir))
found_files = getListOfFiles(args.indir)
found_files.sort()
for f in found_files:
if _is_mseed(f):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to process list.".format(f))
valid_files.append((f, 'MSEED'))
else:
LOGGER.info("{0} is NOT a valid miniSEED file.".format(f))
# take list of valid files and load them into PH5
ph5_object = experiment.ExperimentGroup(nickname=args.nickname,
currentpath=args.ph5path)
ph5_object.ph5open(True)
ph5_object.initgroup()
obs = ObspytoPH5(ph5_object, args.ph5path, args.num_mini, args.first_mini)
if args.verbose:
obs.verbose = True
obs.get_minis(args.ph5path)
if args.num_mini:
total = 0
for entry in valid_files:
total = total + os.path.getsize(entry[0])
obs.mini_size_max = (total * .60) / args.num_mini
index_t_full = list()
for entry in valid_files:
message, index_t = obs.toph5(entry)
for e in index_t:
index_t_full.append(e)
if message == "stop":
LOGGER.error("Stopping program...")
break
if len(obs.time_t) > 0:
LOGGER.info('Populating Time table')
for entry in obs.time_t:
ph5_object.ph5_g_receivers.populateTime_t_(entry)
LOGGER.info("Populating Index table")
for entry in index_t_full:
ph5_object.ph5_g_receivers.populateIndex_t(entry)
obs.update_external_references(index_t_full)
ph5_object.ph5close()
def get_args():
''' Parse input args
-r raw file
-f file containing list of raw files
-n output file
-k kef file # REMOVED
-d dep file # REMOVED
-p print out table list
-M create a specific number of miniPH5 files
-S First index of miniPH5_xxxxx.ph5
'''
global FILES, PH5, SR, WINDOWS, OVERIDE, NUM_MINI, FIRST_MINI
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.usage = ("Version {0} 125a2ph5 [--help][--raw raw_file | "
"--file file_list_file] --nickname output_file_prefix".
format(PROG_VERSION))
parser.description = ("Read a raw texan files and optionally a kef "
"file into ph5 format.")
parser.add_argument("-r",
"--raw",
dest="rawfile",
help="RT-125(a) texan raw file",
metavar="raw_file")
parser.add_argument("-f",
"--file",
dest="infile",
help=("File containing list of RT-125(a) "
"raw file names."),
metavar="file_list_file")
parser.add_argument("-o",
"--overide",
dest="overide",
help="Overide file name checks.",
action="store_true",
default=False)
parser.add_argument("-n",
"--nickname",
dest="outfile",
help="The ph5 file prefix (experiment nick name).",
metavar="output_file_prefix")
parser.add_argument("-M",
"--num_mini",
dest="num_mini",
help="Create given number of miniPH5_xxxxx.ph5 files.",
metavar="num_mini",
type=int,
default=None)
parser.add_argument("-S",
"--first_mini",
dest="first_mini",
help="The index of the first miniPH5_xxxxx.ph5 file.",
metavar="first_mini",
type=int,
default=1)
parser.add_argument("-s",
"--samplerate",
dest="samplerate",
help="Extract only data at given sample rate.",
metavar="samplerate")
parser.add_argument("-w",
"--windows_file",
dest="windows_file",
help=("File containing list of time windows to "
"process. Window start time Window length, "
"seconds\n "
"----------------- ----\n "
"YYYY:JJJ:HH:MM:SS SSSS"),
metavar="windows_file")
parser.add_argument("-p",
help="Do print",
dest="doprint",
action="store_true",
default=False)
args = parser.parse_args()
FILES = []
PH5 = None
OVERIDE = args.overide
SR = args.samplerate
NUM_MINI = args.num_mini
FIRST_MINI = args.first_mini
if args.infile is not None:
read_infile(args.infile)
elif args.rawfile is not None:
FILES.append(args.rawfile)
if args.outfile is not None:
PH5 = args.outfile
if args.doprint is not False:
ex = experiment.ExperimentGroup()
ex.ph5open(True)
ex.initgroup()
keys(ex)
ex.ph5close()
sys.exit()
if args.windows_file is not None:
WINDOWS = read_windows_file(args.windows_file)
else:
WINDOWS = None
if PH5 is None:
LOGGER.error("Missing required option. Try --help\n")
sys.exit()
if not os.path.exists(PH5) and not os.path.exists(PH5 + '.ph5'):
LOGGER.error("{0} not found.".format(PH5))
sys.exit()
else:
# Set up logging
# Write log to file
ch = logging.FileHandler(os.path.join('.', "125a2ph5.log"))
ch.setLevel(logging.INFO)
# Add formatter
formatter = logging.Formatter(LOGGING_FORMAT)
ch.setFormatter(formatter)
LOGGER.addHandler(ch)
def openPH5(filename):
exrec = experiment.ExperimentGroup(nickname=filename)
exrec.ph5open(True)
exrec.initgroup()
return exrec
def get_args():
''' Parse input args
-r raw file
-f file containing list of raw files
-n output file
-k kef file # REMOVED
-d dep file # REMOVED
-p print out table list
-M create a specific number of miniPH5 files
-S First index of miniPH5_xxxxx.ph5
'''
global FILES, PH5, SR, WINDOWS, OVERIDE, NUM_MINI, FIRST_MINI
from optparse import OptionParser
oparser = OptionParser()
oparser.usage = "Version %s 125a2ph5 [--help][--raw raw_file | --file file_list_file] --nickname output_file_prefix" % PROG_VERSION
oparser.description = "Read a raw texan files and optionally a kef file into ph5 format."
oparser.add_option("-r",
"--raw",
dest="rawfile",
help="RT-125(a) texan raw file",
metavar="raw_file")
oparser.add_option(
"-f",
"--file",
dest="infile",
help="File containing list of RT-125(a) raw file names.",
metavar="file_list_file")
oparser.add_option("-o",
"--overide",
dest="overide",
help="Overide file name checks.",
action="store_true",
default=False)
oparser.add_option("-n",
"--nickname",
dest="outfile",
help="The ph5 file prefix (experiment nick name).",
metavar="output_file_prefix")
oparser.add_option(
"-M",
"--num_mini",
dest="num_mini",
help="Create a given number of miniPH5_xxxxx.ph5 files.",
metavar="num_mini",
type='int',
default=None)
oparser.add_option("-S",
"--first_mini",
dest="first_mini",
help="The index of the first miniPH5_xxxxx.ph5 file.",
metavar="first_mini",
type='int',
default=1)
#oparser.add_option ("-d", "--dep", dest = "depfile",
#help = "Rawmeet dep file.",
#metavar = "dep_file")
oparser.add_option("-s",
"--samplerate",
dest="samplerate",
help="Extract only data at given sample rate.",
metavar="samplerate")
oparser.add_option(
"-w",
"--windows_file",
dest="windows_file",
help="File containing list of time windows to process.\n\
Window start time Window length, seconds\n\
----------------- ----\n\
YYYY:JJJ:HH:MM:SS SSSS",
metavar="windows_file")
oparser.add_option("-p",
dest="doprint",
action="store_true",
default=False)
options, args = oparser.parse_args()
#print options.outfile
FILES = []
PH5 = None
KEFFILE = None
DEPFILE = None
OVERIDE = options.overide
SR = options.samplerate
NUM_MINI = options.num_mini
FIRST_MINI = options.first_mini
if options.infile != None:
read_infile(options.infile)
elif options.rawfile != None:
FILES.append(options.rawfile)
if options.outfile != None:
PH5 = options.outfile
if options.doprint != False:
ex = experiment.ExperimentGroup()
ex.ph5open(True)
ex.initgroup()
keys(ex)
ex.ph5close()
sys.exit()
#if options.keffile != None :
#KEFFILE = options.keffile
#if options.depfile != None :
#DEPFILE = options.depfile
if options.windows_file != None:
WINDOWS = read_windows_file(options.windows_file)
else:
WINDOWS = None
if PH5 == None:
#print H5, FILES
sys.stderr.write("Error: Missing required option. Try --help\n")
sys.exit()
if not os.path.exists(PH5) and not os.path.exists(PH5 + '.ph5'):
sys.stderr.write("Error: %s does not exist!\n" % PH5)
sys.exit()
else:
# Set up logging
#if not os.path.exists (OUTPATH) :
#os.mkdir (OUTPATH)
logging.basicConfig(filename=os.path.join('.', "125a2ph5.log"),
format="%(asctime)s %(message)s",
level=logging.INFO)