def main(args):
failed = args.failed
queried = args.queried
searchlist = []
idlist = []
badlist = []
for filename in os.listdir(failed):
print('searching through terminal output %s' % filename)
with open('%s/%s' % (failed, filename)) as fp:
for cnt, line in enumerate(fp):
if line[0:6] == 'Failed':
f, t, g, d, v, o, e, id_no = line.split()
try:
searchobj = get_event_by_id(id_no)
searchlist.append(searchobj)
idlist.append(id_no)
except:
badlist.append(id_no)
print('getting details for %i events: ' % (len(idlist)), idlist)
print('these events %i still failed: ' % (len(badlist)), badlist)
detaildf = get_summary_data_frame(searchlist)
detaildf.to_csv('%s/%sfailed.csv' % (queried, queried),
header=True,
index=False,
na_rep=np.nan)
def test_get_summary_data_frame():
cassettes, datadir = get_datadir()
tape_file = os.path.join(cassettes, 'dataframes_summary.yaml')
with vcr.use_cassette(tape_file):
events = search(starttime=datetime(1994, 6, 1),
endtime=datetime(1994, 10, 6),
minmagnitude=8.0, maxmagnitude=9.0)
df = get_summary_data_frame(events)
assert len(df) == 2
assert df.iloc[0]['magnitude'] == 8.2
def test_get_summary_data_frame():
datadir = get_datadir()
tape_file = os.path.join(datadir, 'vcr_summary_frame.yaml')
with vcr.use_cassette(tape_file):
events = search(starttime=datetime(1994, 6, 1),
endtime=datetime(1994, 10, 6),
minmagnitude=8.0, maxmagnitude=9.0, verbose=True)
df = get_summary_data_frame(events)
assert len(df) == 2
assert df.iloc[0]['magnitude'] == 8.2
def test_get_summary_data_frame():
cassettes, datadir = get_datadir()
tape_file = os.path.join(cassettes, "dataframes_summary.yaml")
with vcr.use_cassette(tape_file, record_mode="new_episodes"):
events = search.search(
starttime=datetime(1994, 6, 1),
endtime=datetime(1994, 10, 6),
minmagnitude=8.0,
maxmagnitude=9.0,
)
df = get_summary_data_frame(events)
assert len(df) == 2
assert df.iloc[0]["magnitude"] == 8.2
def updateSequences(self, stime):
etime = stime + timedelta(days=1)
events = search(starttime=stime,
endtime=etime,
minlatitude=-90,
maxlatitude=90,
minlongitude=-180,
maxlongitude=180,
minmagnitude=0.0,
maxmagnitude=9.9)
todayframe = get_summary_data_frame(events)
todaydata = get_day_counts(GDICT, todayframe)
todaygrid = Grid2D(data=todaydata, geodict=GDICT)
for row in range(0, GDICT.ny):
for col in range(0, GDICT.nx):
if row == 19 and col == 29:
foo = 1
clat, clon = GDICT.getLatLon(row, col)
tvalue = todaygrid._data[row, col]
mvalue = self._meangrid._data[row, col]
svalue = self._stdgrid._data[row, col]
# thresh = tvalue > mvalue + svalue * 3
thresh = tvalue > MINEQ
xmin = clon - GDICT.dx / 2
xmax = clon + GDICT.dx / 2
ymin = clat - GDICT.dy / 2
ymax = clat + GDICT.dy / 2
if thresh:
c1 = todayframe['latitude'] > ymin
c2 = todayframe['latitude'] <= ymax
c3 = todayframe['longitude'] > xmin
c4 = todayframe['longitude'] <= xmax
cluster = todayframe[c1 & c2 & c3 & c4].copy()
class_frame, pproj = self.get_clusters(cluster, clon, clat)
self.insertSequences(class_frame, pproj)
# call a method that filters out clusters that don't match the definition
# of an earthquake sequence.
self.cleanSequences()
def main():
parser = get_parser()
args = parser.parse_args()
# make sure we don't have -e option AND --numdays option
if args.endTime is not None and args.numdays is not None:
msg = ('You must specify end time or number of days since '
'start time, not both. Exiting.')
print(msg)
sys.exit(1)
if not args.endTime and args.numdays:
args.endTime = args.startTime + timedelta(args.numdays)
setup_logger(args.logfile, args.loglevel)
tsum = (args.bounds is not None) + \
(args.radius is not None) + (args.country is not None)
if tsum != 1:
logging.error(
'Please specify a bounding box, radius, or country code.')
sys.exit(1)
latitude = None
longitude = None
radiuskm = None
lonmin = latmin = lonmax = latmax = None
bounds = None
if args.radius:
latitude = args.radius[0]
longitude = args.radius[1]
radiuskm = args.radius[2]
if args.bounds:
lonmin, lonmax, latmin, latmax = args.bounds
# fix longitude bounds when crossing dateline
if lonmin > lonmax and lonmax >= -180:
lonmin -= 360
else:
lonmin, lonmax, latmin, latmax = None, None, None, None
bounds = (lonmin, lonmax, latmin, latmax)
if args.country:
ccode = args.country
if not check_ccode(ccode):
curl = 'https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2'
fmt = ('%s is not a valid ISO 3166 country code. '
'See %s for the list.')
tpl = (ccode, curl)
logging.error(fmt % tpl)
sys.exit(1)
bounds = get_country_bounds(ccode, args.buffer) # this returns a list
minmag = 0.0
maxmag = 9.9
if args.magRange:
minmag = args.magRange[0]
maxmag = args.magRange[1]
minsig = 0
maxsig = 5000
if args.sigRange:
minsig = args.sigRange[0]
maxsig = args.sigRange[1]
if args.getCount:
if isinstance(bounds, tuple) or bounds is None:
nevents = count(starttime=args.startTime,
endtime=args.endTime,
updatedafter=args.after,
minlatitude=latmin,
maxlatitude=latmax,
minlongitude=lonmin,
maxlongitude=lonmax,
latitude=latitude,
longitude=longitude,
maxradiuskm=radiuskm,
catalog=args.catalog,
contributor=args.contributor,
maxmagnitude=maxmag,
minmagnitude=minmag,
minsig=minsig,
maxsig=maxsig,
producttype=args.limitByProductType)
else:
for lonmin, lonmax, latmin, latmax in bounds:
nevents = 0
nevents += count(starttime=args.startTime,
endtime=args.endTime,
updatedafter=args.after,
minlatitude=latmin,
maxlatitude=latmax,
minlongitude=lonmin,
maxlongitude=lonmax,
latitude=latitude,
longitude=longitude,
maxradiuskm=radiuskm,
catalog=args.catalog,
contributor=args.contributor,
minsig=minsig,
maxsig=maxsig,
maxmagnitude=maxmag,
minmagnitude=minmag,
producttype=args.limitByProductType)
print('There are %i events matching input criteria.' % nevents)
sys.exit(0)
if isinstance(bounds, tuple) or bounds is None:
events = search(starttime=args.startTime,
endtime=args.endTime,
updatedafter=args.after,
minlatitude=latmin,
maxlatitude=latmax,
minlongitude=lonmin,
maxlongitude=lonmax,
latitude=latitude,
longitude=longitude,
maxradiuskm=radiuskm,
catalog=args.catalog,
contributor=args.contributor,
maxmagnitude=maxmag,
minmagnitude=minmag,
minsig=minsig,
maxsig=maxsig,
producttype=args.limitByProductType,
host=args.host,
eventtype=args.event_type,
alertlevel=args.alert_level)
else:
events = []
for i, tbounds in enumerate(bounds):
lonmin, lonmax, latmin, latmax = tbounds
fmt = 'Checking bounds %i of %i for %s...\n'
tpl = (i + 1, len(bounds), ccode)
logging.debug(fmt % tpl)
tevents = search(starttime=args.startTime,
endtime=args.endTime,
updatedafter=args.after,
minlatitude=latmin,
maxlatitude=latmax,
minlongitude=lonmin,
maxlongitude=lonmax,
latitude=latitude,
longitude=longitude,
maxradiuskm=radiuskm,
catalog=args.catalog,
contributor=args.contributor,
maxmagnitude=maxmag,
minmagnitude=minmag,
minsig=minsig,
maxsig=maxsig,
producttype=args.limitByProductType,
host=args.host,
eventtype=args.event_type,
alertlevel=args.alert_level)
events += tevents
if not len(events):
logging.info('No events found matching your search criteria. Exiting.')
sys.exit(0)
if (args.getAngles != 'none' or
args.getAllMags or
args.getComponents != 'none'):
logging.info(
'Fetched %i events...creating table.\n' % (len(events)))
supp = args.getMomentSupplement
df = get_detail_data_frame(events, get_all_magnitudes=args.getAllMags,
get_tensors=args.getComponents,
get_focals=args.getAngles,
get_moment_supplement=supp)
else:
logging.info(
'Fetched %i events...creating summary table.\n' % (len(events)))
df = get_summary_data_frame(events)
# order the columns so that at least the initial parameters come the way
# we want them...
first_columns = list(events[0].toDict().keys())
col_list = list(df.columns)
for column in first_columns:
try:
col_list.remove(column)
except Exception as e:
x = 1
df = df[first_columns + col_list]
if args.country:
df = filter_by_country(df, ccode, buffer_km=args.buffer)
logging.info('Created table...saving %i records to %s.\n' %
(len(df), args.filename))
if args.format == 'excel':
df.to_excel(args.filename, index=False)
elif args.format == 'tab':
df.to_csv(args.filename, sep='\t', index=False)
else:
df.to_csv(args.filename, index=False, chunksize=1000)
logging.info('%i records saved to %s.' % (len(df), args.filename))
sys.exit(0)
def updateSequences(self, stime):
etime = stime + timedelta(days=1)
t1 = time.time()
events = search(starttime=stime,
endtime=etime,
minlatitude=-90,
maxlatitude=90,
minlongitude=-180,
maxlongitude=180,
minmagnitude=0.0,
maxmagnitude=9.9)
todayframe = get_summary_data_frame(events)
logging.info('Got day data...')
gdict = GeoDict(self._config['GDICT'])
for row in range(0, gdict.ny):
for col in range(0, gdict.nx):
clat, clon = gdict.getLatLon(row, col)
xmin = clon - gdict.dx / 2
xmax = clon + gdict.dx / 2
ymin = clat - gdict.dy / 2
ymax = clat + gdict.dy / 2
c1 = todayframe['latitude'] > ymin
c2 = todayframe['latitude'] <= ymax
c3 = todayframe['longitude'] > xmin
c4 = todayframe['longitude'] <= xmax
gridframe = todayframe[c1 & c2 & c3 & c4].copy()
if not len(gridframe):
continue
cluster_list, pproj = self.getClusters(gridframe)
if len(cluster_list):
self.insertClusters(cluster_list, pproj)
# now we need to merge sequences that may have fallen on the edge of
# a grid border.
logging.info('Matching sequences...')
sequence_sets = self._seqdb.mergeSequences()
for tseqset in sequence_sets:
seqset = sorted(list(tseqset))
id1 = seqset[0]
frame1 = self._seqdb.getSequenceEvents(id1)
xmin1 = frame1['longitude'].min()
xmax1 = frame1['longitude'].max()
ymin1 = frame1['latitude'].min()
ymax1 = frame1['latitude'].max()
bounds1 = (xmin1, xmax1, ymin1, ymax1)
for i in range(1, len(seqset)):
id2 = seqset[i]
frame2 = self._seqdb.getSequenceEvents(id2)
xmin2 = frame2['longitude'].min()
xmax2 = frame2['longitude'].max()
ymin2 = frame2['latitude'].min()
ymax2 = frame2['latitude'].max()
bounds2 = (xmin2, xmax2, ymin2, ymax2)
logging.info('Merging sequence %i and %i' % (id1, id2))
fmt = 'Bounds1: %.2f, %.2f, %.2f, %.2f'
logging.info(fmt % (bounds1))
fmt = 'Bounds2: %.2f, %.2f, %.2f, %.2f'
logging.info(fmt % (bounds2))
dataframe = pd.concat([frame1, frame2], axis=0)
if dataframe.duplicated('id').any():
foo = 1
proj = self.getProj(dataframe)
sqstats = self.getSequenceStats(dataframe, proj)
self._seqdb.updateSequence(frame2, sqstats, id1)
self._seqdb.deleteSequence(id2)
logging.info('Finding stopped sequences...')
# now find all sequences that seem to have ended (using config criteria)
ended = self._seqdb.getStoppedDataframe()
if len(ended) > 0:
x = 1
self._seqdb.markStoppedSequences(ended)
logging.info('Finding non sequences...')
# find all ended sequences that don't match our criteria for a sequence
deletes = self._seqdb.getNonSequences()
if len(deletes) > 0:
x = 1
self._seqdb.deleteNonSequences(deletes)
t2 = time.time()
