def is_valid_mseed(file_path):
"""
Return True if path is an existing regular file and a valid mseed. False otherwise.
:param file_path: The full file's path.
:return: True if path is an existing regular file and a valid mseed. False otherwise.
"""
return os.path.isfile(file_path) and _is_mseed(file_path)
def get_st(self, data_path):
obsfiles = self.list_directory(data_path)
traces = []
for paths in obsfiles:
if _is_mseed(paths):
st = read(paths)
tr = st[0]
traces.append(tr)
st = Stream(traces)
return st
def create_dict(self, **kwargs):
net_list = kwargs.pop('net_list').split(',')
sta_list = kwargs.pop('sta_list').split(',')
chn_list = kwargs.pop('chn_list').split(',')
obsfiles = []
for top_dir, sub_dir, files in os.walk(self.files_path):
for file in files:
obsfiles.append(os.path.join(top_dir, file))
#obsfiles = [f for f in listdir(self.files_path) if isfile(join(self.files_path, f))]
obsfiles.sort()
data_map = {}
data_map['nets'] = {}
size = 0
for paths in obsfiles:
#paths = os.path.join(self.files_path, i)
if _is_mseed(paths):
header = read(paths, headlonly=True)
net = header[0].stats.network
network = {net: {}}
sta = header[0].stats.station
stations = {sta: {}}
chn = header[0].stats.channel
## Filter per nets
# 1. Check if the net exists, else add
if net not in data_map['nets']:
# 1.1 Check the filter per network
if net in net_list:
data_map['nets'].update(network)
# 1.2 the filter per network is not activated
if net_list[0] == "":
data_map['nets'].update(network)
# 2. Check if the station exists, else add
try:
if sta not in data_map['nets'][net]:
if sta in sta_list:
data_map['nets'][net].update(stations)
if sta_list[0] == "":
data_map['nets'][net].update(stations)
except:
pass
# 3. Check if the channels exists, else add
try:
if chn in data_map['nets'][net][sta]:
if chn in chn_list:
data_map['nets'][net][sta][chn].append(paths)
size = size + 1
if chn_list[0] == "":
data_map['nets'][net][sta][chn].append(paths)
size = size + 1
else:
if chn in chn_list:
data_map['nets'][net][sta][chn] = [paths]
size = size + 1
if chn_list[0] == "":
data_map['nets'][net][sta][chn] = [paths]
size = size + 1
except:
pass
return data_map, size
def main():
args = get_args(sys.argv[1:])
if args.nickname[-3:] == 'ph5':
ph5file = os.path.join(args.ph5path, args.nickname)
else:
ph5file = os.path.join(args.ph5path, args.nickname + '.ph5')
args.nickname += '.ph5'
PATH = os.path.dirname(args.ph5path) or '.'
# Debugging
os.chdir(PATH)
# Write log to file
ch = logging.FileHandler(os.path.join(".", "datatoph5.log"))
ch.setLevel(logging.INFO)
# Add formatter
formatter = logging.Formatter(LOGGING_FORMAT)
ch.setFormatter(formatter)
LOGGER.addHandler(ch)
if not os.path.exists(ph5file):
LOGGER.warning("{0} not found. Creating...".format(ph5file))
# Create ph5 file
ex = experiment.ExperimentGroup(nickname=ph5file)
ex.ph5open(True) # Open ph5 file for editing
ex.initgroup()
# Update /Experiment_g/Receivers_g/Receiver_t
default_receiver_t = initialize_ph5.create_default_receiver_t()
initialize_ph5.set_receiver_t(default_receiver_t)
LOGGER.info(
"Removing temporary {0} kef file.".format(default_receiver_t))
os.remove(default_receiver_t)
ex.ph5close()
LOGGER.info("Done... Created new PH5 file {0}.".format(ph5file))
# Produce list of valid files from file, list, and dirs
valid_files = list()
if args.rawfile:
if _is_mseed(args.rawfile):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to process list.".format(args.rawfile))
valid_files.append((args.rawfile, 'MSEED'))
else:
LOGGER.info("{0} is a NOT valid miniSEED file.".format(
args.rawfile))
try:
if _is_mseed(args.infile):
LOGGER.error("The given list file is a miniseed file. You have "
"been confused between two option -r and -f.")
sys.exit()
except TypeError:
pass
if args.infile:
LOGGER.info("Checking list...")
with open(args.infile) as f:
content = f.readlines()
for line in content:
if os.path.isfile(line.rstrip()):
if _is_mseed(line.rstrip()):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to"
"process list.".format(line.rstrip()))
valid_files.append((line.rstrip(), 'MSEED'))
else:
LOGGER.info("{0} is NOT a valid miniSEED "
"file.".format(line.rstrip()))
else:
LOGGER.info("{0} does not exist".format(line.rstrip()))
if args.indir:
LOGGER.info("Scanning directory {0} and sub directories...".format(
args.indir))
found_files = getListOfFiles(args.indir)
found_files.sort()
for f in found_files:
if _is_mseed(f):
LOGGER.info("{0} is a valid miniSEED file. "
"Adding to process list.".format(f))
valid_files.append((f, 'MSEED'))
else:
LOGGER.info("{0} is NOT a valid miniSEED file.".format(f))
# take list of valid files and load them into PH5
ph5_object = experiment.ExperimentGroup(nickname=args.nickname,
currentpath=args.ph5path)
ph5_object.ph5open(True)
ph5_object.initgroup()
obs = ObspytoPH5(ph5_object, args.ph5path, args.num_mini, args.first_mini)
if args.verbose:
obs.verbose = True
obs.get_minis(args.ph5path)
if args.num_mini:
total = 0
for entry in valid_files:
total = total + os.path.getsize(entry[0])
obs.mini_size_max = (total * .60) / args.num_mini
index_t_full = list()
for entry in valid_files:
message, index_t = obs.toph5(entry)
for e in index_t:
index_t_full.append(e)
if message == "stop":
LOGGER.error("Stopping program...")
break
if len(obs.time_t) > 0:
LOGGER.info('Populating Time table')
for entry in obs.time_t:
ph5_object.ph5_g_receivers.populateTime_t_(entry)
LOGGER.info("Populating Index table")
for entry in index_t_full:
ph5_object.ph5_g_receivers.populateIndex_t(entry)
obs.update_external_references(index_t_full)
ph5_object.ph5close()
def create_dict(self, **kwargs):
self.send_message.emit("Creating Dictionary")
net_list = kwargs.pop('net_list', "").split(',')
sta_list = kwargs.pop('sta_list', "").split(',')
chn_list = kwargs.pop('chn_list', "").split(',')
obsfiles = self.list_directory()
data_map = {}
info = {}
data_map['nets'] = {}
size = 0
for paths in obsfiles:
if _is_mseed(paths):
header = read(paths, headlonly=True)
net = header[0].stats.network
network = {net: {}}
sta = header[0].stats.station
stations = {sta: {}}
chn = header[0].stats.channel
starttime_ini = header[0].stats.starttime
endtime_ini = header[0].stats.endtime
meta = [net, sta, chn]
key_meta = meta[0] + meta[1] + meta[2]
## Filter per nets
# 1. Check if the net exists, else add
if net not in data_map['nets']:
# 1.1 Check the filter per network
if net in net_list:
data_map['nets'].update(network)
# 1.2 the filter per network is not activated
if net_list[0] == "":
data_map['nets'].update(network)
# 2. Check if the station exists, else add
try:
if sta not in data_map['nets'][net]:
if sta in sta_list:
data_map['nets'][net].update(stations)
if sta_list[0] == "":
data_map['nets'][net].update(stations)
except:
pass
# 3. Check if the channels exists, else add
try:
# 3.1 if already exists just add
if chn in data_map['nets'][net][sta]:
if chn in chn_list:
data_map['nets'][net][sta][chn].append(paths)
size = size + 1
if endtime_ini - info[key_meta][0][1] > 0:
info[key_meta][0][1] = endtime_ini
elif info[key_meta][0][0] - starttime_ini > 0:
info[key_meta][0][0] = starttime_ini
if chn_list[0] == "":
data_map['nets'][net][sta][chn].append(paths)
if endtime_ini - info[key_meta][0][1] > 0:
info[key_meta][0][1] = endtime_ini
elif info[key_meta][0][0] - starttime_ini > 0:
info[key_meta][0][0] = starttime_ini
size = size + 1
else:
# 3.2 if does't exist create a list
if chn in chn_list:
data_map['nets'][net][sta][chn] = [meta, paths]
starttime_chn = starttime_ini
endtime_chn = endtime_ini
info[key_meta] = [[starttime_chn, endtime_chn],
self.metadata.select(channel=chn,
station=sta)
]
size = size + 1
if chn_list[0] == "":
data_map['nets'][net][sta][chn] = [meta, paths]
starttime_chn = starttime_ini
endtime_chn = endtime_ini
info[key_meta] = [[starttime_chn, endtime_chn],
self.metadata.select(channel=chn,
station=sta)
]
size = size + 1
except:
pass
self.send_message.emit("Ended Dictionary")
return data_map, size, info
def data_availability(cls, files_path: str, only_this=True):
import matplotlib.pyplot as plt
import matplotlib.dates as mdt
from isp.Gui.Frames import MatplotlibFrame
fig, hax = plt.subplots(1, 1, figsize=(12, 6))
cls.mpf = MatplotlibFrame(fig)
starttimes = []
endtimes = []
if only_this:
obsfiles = [
f for f in listdir(files_path) if isfile(join(files_path, f))
]
obsfiles.sort()
else:
obsfiles = []
for top_dir, sub_dir, files in os.walk(files_path):
for file in files:
obsfiles.append(os.path.join(top_dir, file))
obsfiles.sort()
data_map = {}
data_map['nets'] = {}
for i in obsfiles:
paths = os.path.join(files_path, i)
if _is_mseed(paths):
header = read(paths, headlonly=True)
gap = header.get_gaps()
net = header[0].stats.network
sta = header[0].stats.station
chn = header[0].stats.channel
# times
starttimes.append(header[0].stats.starttime)
start = header[0].stats.starttime.matplotlib_date
endtimes.append(header[0].stats.endtime)
end = header[0].stats.endtime.matplotlib_date
name = net + "." + sta + "." + chn
hax.hlines(name,
start,
end,
colors='k',
linestyles='solid',
label=name,
lw=2)
if len(gap) > 0:
for i in range(len(gap)):
starttime_gap = gap[i][4].matplotlib_date
endtime_gap = gap[i][5].matplotlib_date
hax.hlines(name,
starttime_gap,
endtime_gap,
colors='r',
linestyles='solid',
label=name,
lw=2)
start_time = min(starttimes)
end_time = max(endtimes)
formatter = mdt.DateFormatter('%y/%m/%d/%H:%M:%S.%f')
hax.xaxis.set_major_formatter(formatter)
hax.set_xlabel("Date")
hax.set_xlim(start_time.matplotlib_date, end_time.matplotlib_date)
cls.mpf.show()
def create_dict(self, **kwargs):
net_list = kwargs.pop('net_list', "").split(',')
sta_list = kwargs.pop('sta_list', "").split(',')
chn_list = kwargs.pop('chn_list', "").split(',')
obsfiles = self.list_directory(self.data_path)
data_map = {}
data_map['area_coords'] = self.area_coords
data_map['dx'] = self.dx
data_map['dy'] = self.dy
data_map['depth'] = self.depth
data_map['nets'] = {}
size = 0
for paths in obsfiles:
if _is_mseed(paths):
header = read(paths, headlonly=True)
net = header[0].stats.network
network = {net: {}}
sta = header[0].stats.station
stations = {sta: {}}
chn = header[0].stats.channel
meta = [net, sta, chn]
id = header[0].id
print(id)
## Filter per nets
# 1. Check if the net exists, else add
if net not in data_map['nets']:
# 1.1 Check the filter per network
if net in net_list:
data_map['nets'].update(network)
# 1.2 the filter per network is not activated
if net_list[0] == "":
data_map['nets'].update(network)
# 2. Check if the station exists, else add
try:
if sta not in data_map['nets'][net]:
if sta in sta_list:
data_map['nets'][net].update(stations)
if sta_list[0] == "":
data_map['nets'][net].update(stations)
except:
pass
# 3. Check if the channels exists, else add
# 3.1 if already exists just add
if chn in data_map['nets'][net][sta]:
if chn in chn_list:
sta_coords = self.inventory.get_coordinates(id)
time_mat = self.griding(self.area_coords, sta_coords,
self.dx, self.dy, self.depth)
data_map['nets'][net][sta][chn].append(time_mat)
size = size + 1
if chn_list[0] == "":
sta_coords = self.inventory.get_coordinates(id)
time_mat = self.griding(self.area_coords, sta_coords,
self.dx, self.dy, self.depth)
data_map['nets'][net][sta][chn].append(time_mat)
size = size + 1
else:
# 3.2 if does't exist create a list
if chn in chn_list:
sta_coords = self.inventory.get_coordinates(id)
time_mat = backproj.griding(self.area_coords,
sta_coords, self.dx,
self.dy, self.depth)
data_map['nets'][net][sta][chn] = [time_mat]
size = size + 1
if chn_list[0] == "":
sta_coords = self.inventory.get_coordinates(id)
time_mat = backproj.griding(self.area_coords,
sta_coords, self.dx,
self.dy, self.depth)
data_map['nets'][net][sta][chn] = [time_mat]
size = size + 1
return data_map
# problems_db[key].append(("station_mismatch", value['station'], value['path']))
# get the component name from header
component = value['component']
#check if the component matches the filename
if not key.split('.')[1] == component:
temp_dict["problem"]["component_mismatch"].append(
(component, value['path']))
# problems_db[key].append(("component_mismatch", component, value['path']))
if len(key) < 15:
# bad filename.
# see if it is miniseed
bool_seed = _is_mseed(join(value['path'], key))
temp_dict["problem"]["bad_filename"].append((bool_seed, value['path']))
# problems_db[key].append(("bad_filename", bool_seed, value['path']))
else:
# check if the starttime matches the filename
nt = make_datetime(key)
ht = UTCDateTime(value['starttime'])
# if not (nt.year == ht.year and nt.month == ht.month and nt.day == ht.day and nt.hour==ht.hour):# and nt.minute==ht.minute and nt.second==ht.second):
if (120 <= abs(ht - nt)):
# if not (value['station'] == 'GA4'):
# continue
# print '______________'
# print(value['path'], key, value['station'], nt, ht)
# print(nt-ht)