def make_heartbeat(self, session=None): ''' Make an Event row that will only trigger a notification for groups with a heartbeat group specification ''' last_hb = session.query(Event).filter(Event.event_id == 'heartbeat').all() make_hb = False if last_hb: if time.time() > (last_hb[-1].time) + 24*60*60: make_hb = True else: make_hb = True if make_hb is True: e = Event() e.time = time.time() e.event_id = 'heartbeat' e.magnitude = 10 e.lat = 0 e.lon = 0 e.title = 'ShakeCast Heartbeat' e.place = 'ShakeCast is running' e.status = 'new' e.directory_name = os.path.join(self.data_dir, e.event_id) session.add(e) session.commit() self.get_event_map(e)
def make_heartbeat(self, session=None): ''' Make an Event row that will only trigger a notification for groups with a heartbeat group specification ''' last_hb = session.query(Event).filter( Event.event_id == 'heartbeat').all() make_hb = False if last_hb: if time.time() > (last_hb[-1].time) + 24 * 60 * 60: make_hb = True else: make_hb = True if make_hb is True: e = Event(event_id='heartbeat', save=True) e.time = time.time() e.magnitude = 10 e.lat = 0 e.lon = 0 e.title = 'ShakeCast Heartbeat' e.place = 'ShakeCast is running' e.status = 'new' e.type = 'heartbeat' session.add(e) session.commit() self.get_event_map(e)
def grab_from_directory(directory, session=None): info_loc = os.path.join(directory, 'info.json') error = '' log = '' try: with open(info_loc, 'r') as info_file: info = json.loads(info_file.read()) # Load shakemap grid to get extra info grid_loc = os.path.join(directory, 'grid.xml') grid = ShakeMapGrid() grid.load(grid_loc) except Exception as e: error = str(e) log = error event_info = info['input']['event_information'] # make timestamp dt = datetime.datetime.strptime(event_info['origin_time'], '%Y-%d-%mT%H:%M:%SZ') timestamp = time.mktime(dt.timetuple()) event = Event(status='new', event_id=grid.event_id, title='M {} - {}'.format(event_info['magnitude'], event_info['location']), place=event_info['location'], time=timestamp, magnitude=event_info['magnitude'], lon=event_info['longitude'], lat=event_info['latitude'], depth=event_info['depth']) session.add(event) proc = info['processing'] shakemap = ShakeMap( status='new', event=event, shakemap_id=grid.event_id, lat_min=grid.lat_min, lat_max=grid.lat_max, lon_min=grid.lon_min, lon_max=grid.lon_max, generation_timestamp=proc['shakemap_versions']['process_time'], recieve_timestamp=time.time()) session.add(shakemap) session.commit() return { 'status': 'finished', 'error': error, 'log': log, 'message': 'File scrape for new earthquakes' }
def _refresh_from_file(file, src, sess): logging.debug(file.path) new_events = [] logfile = sess.query(Logfile).get(file.path) if logfile == None: logfile = Logfile(path=file.path, offset=0) sess.add(logfile) with open(logfile.path, 'rb') as f: logging.debug('offset: {}'.format(logfile.offset)) f.seek(logfile.offset) iter = 0 for line in f: try: data = utils.logline_to_dict(line.decode()) if not ('type' in data and data['type'] == 'crash'): if 'milestone' in data: event = Event(type=EventType.milestone, data=json.dumps(data), time=utils.crawl_date_to_datetime( data['time']), src_abbr=src.name) else: event = Event(type=EventType.game, data=json.dumps(data), time=utils.crawl_date_to_datetime( data['end']), src_abbr=src.name) sess.add(event) if len( new_events ) < 100: # don't want to do huge sends over sockets TODO: make a config option new_events.append(event) except KeyError as e: logging.error('key {} not found'.format(e)) except Exception as e: # how scandalous! Don't want one broken line to break everything logging.exception( 'Something unexpected happened, skipping this event') iter += 1 logfile.offset += len(line) if iter % 1000 == 0: # don't spam commits sess.commit() logfile.offset = f.tell() sess.commit() return new_events
def get_new_events(self, session=None, scenario=False): """ Checks the json feed for new earthquakes """ sc = SC() event_str = '' new_events = [] for eq_id in self.earthquakes.keys(): eq = self.earthquakes[eq_id] # ignore info from unfavorable networks and low mag eqs if (eq['properties']['net'] in self.ignore_nets or eq['properties']['mag'] < sc.new_eq_mag_cutoff): continue # get event id and all ids event = Event() event.all_event_ids = eq['properties']['ids'] if scenario is False: event.event_id = eq_id else: event.event_id = eq_id + '_scenario' event.all_event_ids = event.event_id # use id and all ids to determine if the event is new and # query the old event if necessary old_shakemaps = [] old_notifications = [] if event.is_new() is False: event.status = 'processed' ids = event.all_event_ids.strip(',').split(',') old_events = [(session.query(Event) .filter(Event.event_id == each_id) .first()) for each_id in ids] # remove older events for old_event in old_events: if old_event is not None: old_notifications += old_event.notifications old_shakemaps += old_event.shakemaps # if one of these old events hasn't had # notifications sent, this event should be sent if old_event.status == 'new': event.status = 'new' session.delete(old_event) else: event.status = 'new' # over ride new status if scenario if scenario is True: event.status = 'scenario' # Fill the rest of the event info event.directory_name = os.path.join(self.data_dir, event.event_id) event.title = self.earthquakes[eq_id]['properties']['title'] event.place = self.earthquakes[eq_id]['properties']['place'] event.time = self.earthquakes[eq_id]['properties']['time']/1000.0 event.magnitude = eq['properties']['mag'] event_coords = self.earthquakes[eq_id]['geometry']['coordinates'] event.lon = event_coords[0] event.lat = event_coords[1] event.depth = event_coords[2] # determine whether or not an event should be kept # based on group definitions keep_event = False groups = session.query(Group).all() if len(groups) > 0: for group in groups: if group.point_inside(event): keep_event = True else: keep_event = True if keep_event is False: continue if old_shakemaps: event.shakemaps = old_shakemaps if old_notifications: event.notifications = old_notifications session.add(event) session.commit() self.get_event_map(event) # add the event to the return list and add info to the # return string new_events += [event] event_str += 'Event: %s\n' % event.event_id # print event_str return new_events, event_str
def make_heartbeat(self): ''' Make an Event row that will only trigger a notification for groups with a heartbeat group_specification ''' session = Session() last_hb = session.query(Event).filter( Event.event_id == 'heartbeat').all() make_hb = False if last_hb: if time.time() > (last_hb[-1].time) + 24 * 60 * 60: make_hb = True else: make_hb = True if make_hb is True: e = Event() e.time = time.time() e.event_id = 'heartbeat' e.magnitude = 10 e.lat = 1000 e.lon = 1000 e.title = 'ShakeCast Heartbeat' e.place = 'ShakeCast is running' e.status = 'new' e.directory_name = os.path.join(self.data_dir, e.event_id) session.add(e) session.commit() self.get_event_map(e) Session.remove()
def get_new_events(self, scenario=False): """ Checks the json feed for new earthquakes """ session = Session() sc = SC() event_str = '' new_events = [] for eq_id in self.earthquakes.keys(): eq = self.earthquakes[eq_id] # ignore info from unfavorable networks and low mag eqs if (eq['properties']['net'] in self.ignore_nets or eq['properties']['mag'] < sc.new_eq_mag_cutoff): continue # get event id and all ids event = Event() event.all_event_ids = eq['properties']['ids'] if scenario is False: event.event_id = eq_id else: event.event_id = eq_id + '_scenario' event.all_event_ids = event.event_id # use id and all ids to determine if the event is new and # query the old event if necessary old_shakemaps = [] old_notifications = [] if event.is_new() is False: event.status = 'processed' ids = event.all_event_ids.strip(',').split(',') old_events = [(session.query(Event).filter( Event.event_id == each_id).first()) for each_id in ids] # remove older events for old_event in old_events: if old_event is not None: old_notifications += old_event.notifications old_shakemaps += old_event.shakemaps # if one of these old events hasn't had # notifications sent, this event should be sent if old_event.status == 'new': event.status = 'new' session.delete(old_event) else: event.status = 'new' # over ride new status if scenario if scenario is True: event.status = 'scenario' # Fill the rest of the event info event.directory_name = os.path.join(self.data_dir, event.event_id) event.title = self.earthquakes[eq_id]['properties']['title'] event.place = self.earthquakes[eq_id]['properties']['place'] event.time = self.earthquakes[eq_id]['properties']['time'] / 1000.0 event.magnitude = eq['properties']['mag'] event_coords = self.earthquakes[eq_id]['geometry']['coordinates'] event.lon = event_coords[0] event.lat = event_coords[1] event.depth = event_coords[2] if old_shakemaps: event.shakemaps = old_shakemaps if old_notifications: event.notifications = old_notifications session.add(event) session.commit() self.get_event_map(event) # add the event to the return list and add info to the # return string new_events += [event] event_str += 'Event: %s\n' % event.event_id Session.remove() print event_str return new_events, event_str
def get_new_events(self, session=None, scenario=False): """ Checks the json feed for new earthquakes """ sc = SC() event_str = '' new_events = [] for eq_id in self.earthquakes.keys(): eq = self.earthquakes[eq_id] # ignore info from unfavorable networks and low mag eqs if (eq['properties']['net'] in self.ignore_nets or eq['properties']['mag'] < sc.new_eq_mag_cutoff): continue # get event id and all ids event = Event() event.all_event_ids = eq['properties']['ids'] if scenario is False: event.event_id = eq_id else: event.event_id = eq_id + '_scenario' event.all_event_ids = event.event_id # use id and all ids to determine if the event is new and # query the old event if necessary old_shakemaps = [] old_notifications = [] if event.is_new() is False: event.status = 'processed' ids = event.all_event_ids.strip(',').split(',') old_events = [(session.query(Event).filter( Event.event_id == each_id).first()) for each_id in ids] # remove older events for old_event in old_events: if old_event is not None: old_notifications += old_event.notifications old_shakemaps += old_event.shakemaps # if one of these old events hasn't had # notifications sent, this event should be sent if old_event.status == 'new': event.status = 'new' session.delete(old_event) else: event.status = 'new' # Fill the rest of the event info event.title = self.earthquakes[eq_id]['properties']['title'] event.place = self.earthquakes[eq_id]['properties']['place'] event.time = self.earthquakes[eq_id]['properties']['time'] / 1000.0 event.magnitude = eq['properties']['mag'] event_coords = self.earthquakes[eq_id]['geometry']['coordinates'] event.lon = event_coords[0] event.lat = event_coords[1] event.depth = event_coords[2] event.type = 'scenario' if scenario is True else 'event' # determine whether or not an event should be kept # based on group definitions. Should always be true for scenario runs keep_event = scenario groups = session.query(Group).all() if len(groups) > 0: for group in groups: if group.point_inside(event): keep_event = True else: keep_event = True if keep_event is False: continue if old_shakemaps: event.shakemaps = old_shakemaps if old_notifications: event.notifications = old_notifications session.add(event) session.commit() self.get_event_map(event) # add the event to the return list and add info to the # return string new_events += [event] event_str += 'Event: %s\n' % event.event_id # print event_str return new_events, event_str
def grab_from_directory(directory, session=None): """ Import an event and shakemap from a local directory There are no naming convensions for the directory, but it must contain an info.json and grid.xml shakemap product. Event and ShakeMap database entries will be created as Scenario products and the event can be triggered through the UI as a scenario """ info_loc = os.path.join(directory, 'info.json') error = '' log = '' try: with open(info_loc, 'r') as info_file: info = json.loads(info_file.read()) # Load shakemap grid to get extra info grid_loc = os.path.join(directory, 'grid.xml') grid = ShakeMapGrid() grid.load(grid_loc) except Exception as e: error = str(e) log = error raise (e) event_info = info['input']['event_information'] # make timestamp try: dt = datetime.datetime.strptime(event_info['origin_time'], '%Y-%d-%mT%H:%M:%SZ') timestamp = time.mktime(dt.timetuple()) except Exception: # can't parse the timestamp... just use current time timestamp = time.time() event = Event(status='scenario', event_id=grid.event_id, title='M {} - {}'.format(event_info['magnitude'], event_info['location']), place=event_info['location'], time=timestamp, magnitude=event_info['magnitude'], lon=event_info['longitude'], lat=event_info['latitude'], depth=event_info['depth'], override_directory=directory, type='scenario', save=True) session.add(event) proc = info['processing'] shakemap = ShakeMap( status='scenario', event=event, shakemap_id=grid.event_id, lat_min=grid.lat_min, lat_max=grid.lat_max, lon_min=grid.lon_min, lon_max=grid.lon_max, generation_timestamp=proc['shakemap_versions']['process_time'], recieve_timestamp=time.time(), override_directory=directory, shakemap_version=proc['shakemap_versions']['map_version'], type='scenario', save=True) session.add(shakemap) session.commit() return event, shakemap