def query(self,start_time=datetime.datetime(1800,1,1),end_time=datetime.datetime.utcnow(), mag_threshold=0.0,alert_threshold='green',version='last',eventid=None): levels = {'green':0, 'yellow':1, 'orange':2, 'red':3} if eventid is not None: all_event_folders = [self.getEventFolder(eventid)] version = 'all' else: all_event_folders = self.getAllEventFolders() event_data = [] do_process_time = False if eventid is not None: do_process_time = True df = pd.DataFrame(columns=PagerData.getSeriesColumns(processtime=do_process_time)) jsonfolders = [] for event_folder in all_event_folders: vnums = self.getVersionNumbers(event_folder) if version == 'first': vnum = vnums[0] jsonfolders.append(os.path.join(event_folder,'version.%03d' % vnum,'json')) elif version == 'last': vnum = vnums[-1] jsonfolders.append(os.path.join(event_folder,'version.%03d' % vnum,'json')) elif version == 'eight': for vnum in vnums: jsonfolder = os.path.join(event_folder,'version.%03d' % vnum,'json') pdata = PagerData() try: pdata.loadFromJSON(jsonfolder) except: continue if pdata.processing_time >= pdata.time + datetime.timedelta(seconds=EIGHT_HOURS): break jsonfolders.append(jsonfolder) elif version == 'all': for vnum in vnums: jsonfolder = os.path.join(event_folder,'version.%03d' % vnum,'json') jsonfolders.append(jsonfolder) else: raise PagerException('version option "%s" not supported.' % version) broken = [] for jsonfolder in jsonfolders: pdata = PagerData() vnum = 1000 while vnum > 1: try: pdata.loadFromJSON(jsonfolder) vnum = 0 except: #handle the case where the most recent version of the event has some #sort of error causing it to miss root,jsonfolder = os.path.split(jsonfolder) root2,vfolder = os.path.split(root) vt,vnums = vfolder.split('.') vnum = int(vnums) - 1 jsonfolder = os.path.join(root2,'%s.%03d' % (vt,vnum),'json') if not pdata._is_validated: broken.append(jsonfolder) meetsLevel = levels[pdata.summary_alert] >= levels[alert_threshold] meetsMag = pdata.magnitude >= mag_threshold if pdata.time >= start_time and pdata.time <= end_time and meetsLevel and meetsMag: row = pdata.toSeries(processtime=do_process_time) df = df.append(row,ignore_index=True) df.Version = df.Version.astype(int) df.Elapsed = df.Elapsed.astype(int) df = df.sort_values('EventTime') df = df.set_index('EventID') return (df,broken)
def query(self, start_time=datetime.datetime(1800, 1, 1), end_time=datetime.datetime.utcnow(), mag_threshold=0.0, alert_threshold='green', version='last', eventid=None): """Query PAGER file for events matching input parameters. :param start_time: Datetime indicating the minimum date/time for the search. :param end_time: Datetime indicating the maximum date/time for the search. :param mag_thresh: Minimum magnitude threshold. :param alert_threshold: Minimum alert level threshold ('green','yellow','orange','red'). :param version: Which version(s) to select from events: - 'all' Get all versions. - 'last' Get last version. - 'eight' Get first version that was created more than 8 hours after origin time. :param eventid: Return version(s) for specific event ID. :returns: Pandas dataframe containing columns: - 'EventID' - event ID - 'Impacted Country ($)' Country with largest dollar losses. - 'Version' - Version number - 'EventTime' - Origin Time - 'Lat' - Origin latitude. - 'Lon' - Origin longitude. - 'Depth' - Origin depth. - 'Mag' - Event magnitude. - 'MaxMMI' - Maximum MMI value (felt by at least 1000 people) - 'FatalityAlert' - Fatality alert level ('green','yellow','orange','red') - 'EconomicAlert' - Economic alert level ('green','yellow','orange','red') - 'SummaryAlert' - Summary alert level ('green','yellow','orange','red') - 'Elapsed' - Elapsed time (minutes) between origin time and version. """ levels = {'green': 0, 'yellow': 1, 'orange': 2, 'red': 3} if eventid is not None: all_event_folders = [self.getEventFolder(eventid)] version = 'all' else: all_event_folders = self.getAllEventFolders() event_data = [] do_process_time = False if eventid is not None: do_process_time = True df = pd.DataFrame(columns=PagerData.getSeriesColumns( processtime=do_process_time)) jsonfolders = [] for event_folder in all_event_folders: vnums = self.getVersionNumbers(event_folder) if version == 'first': vnum = vnums[0] jsonfolders.append(os.path.join( event_folder, 'version.%03d' % vnum, 'json')) elif version == 'last': vnum = vnums[-1] jsonfolders.append(os.path.join( event_folder, 'version.%03d' % vnum, 'json')) elif version == 'eight': for vnum in vnums: jsonfolder = os.path.join( event_folder, 'version.%03d' % vnum, 'json') pdata = PagerData() try: pdata.loadFromJSON(jsonfolder) except: continue if pdata.processing_time >= pdata.time + datetime.timedelta(seconds=EIGHT_HOURS): break jsonfolders.append(jsonfolder) elif version == 'all': for vnum in vnums: jsonfolder = os.path.join( event_folder, 'version.%03d' % vnum, 'json') jsonfolders.append(jsonfolder) else: raise PagerException( 'version option "%s" not supported.' % version) broken = [] for jsonfolder in jsonfolders: pdata = PagerData() vnum = 1000 while vnum > 1: try: pdata.loadFromJSON(jsonfolder) vnum = 0 except: # handle the case where the most recent version of the event has some # sort of error causing it to miss root, jsonfolder = os.path.split(jsonfolder) root2, vfolder = os.path.split(root) vt, vnums = vfolder.split('.') vnum = int(vnums) - 1 jsonfolder = os.path.join( root2, '%s.%03d' % (vt, vnum), 'json') if not pdata._is_validated: broken.append(jsonfolder) try: meetsLevel = levels[pdata.summary_alert] >= levels[alert_threshold] except Exception as e: x = 1 meetsMag = pdata.magnitude >= mag_threshold if pdata.time >= start_time and pdata.time <= end_time and meetsLevel and meetsMag: row = pdata.toSeries(processtime=do_process_time) df = df.append(row, ignore_index=True) df.Version = df.Version.astype(int) df.Elapsed = df.Elapsed.astype(int) df = df.sort_values('EventTime') df = df.set_index('EventID') return (df, broken)