def _check_and_get_until( self, _request=None, ): if _request is None: _request = bottle.request until = _request.query.until if until != '': try: until = util.utc_from_iso8601(until) until = util.utc_to_local(until, naive=True) except ValueError: send_error( code=400, message='Invalid until parameter value', ) else: query = OrderedDict([ ('facebook.start_time', OrderedDict([ ('$lte', until), ]), ), ]) return query return None
def _save_events( events_coll, expired_coll, owners_coll, owner_ids, graph ): # Don't waste a call to the Facebook Graph API if not owner_ids: return batch = [ OrderedDict([ ('method', 'GET'), ('relative_url', '{owner_id}/events?date_format=c'.format( owner_id=owner_id, ), ), ]) for owner_id in owner_ids ] reponses = graph.batch(batch) now = datetime.utcnow() for owner_id,response in zip(owner_ids,reponses): if isinstance(response, FacepyError): _mark_as_failed( owners_coll=owners_coll, owner_id=owner_id, now=now, reason=str(response), ) continue # Object does not exist anymore if response is False: _mark_as_failed( owners_coll=owners_coll, owner_id=owner_id, now=now, reason='False response', ) continue if response is None: # None has special significance in mongodb searches # so use 'null' instead. _mark_as_failed( owners_coll=owners_coll, owner_id=owner_id, now=now, reason='Null response', ) continue for event in response['data']: _id = event.pop('id') # TODO find a more efficient way to do this if (events_coll.find_one({'_id': _id}) or expired_coll.find_one({'_id': _id}) ): continue try: # This information is not complete. Save it for now # but allow it to be replaced with more detailed # information later. # TODO what happens if this overwrites good values? # Is that OK? Is it bad? # The check above does not guarantee some other # process won't update this same event before we get # here save = OrderedDict([ ('facebook', event), ('ubernear', OrderedDict([ ('source', 'facebook'), ('fetched', now), ]), ), ]) save['facebook']['start_time'] = utc_from_iso8601( save['facebook']['start_time'], naive=True, ) save['facebook']['end_time'] = utc_from_iso8601( save['facebook']['end_time'], naive=True, ) if 'updated_time' in save['facebook']: save['facebook']['updated_time'] = utc_from_iso8601( save['facebook']['updated_time'], naive=True, ) mongo.save_no_replace( events_coll, _id=_id, save=save, ) except Exception: log.error( 'Could not save event {_id}'.format( _id=_id, ) ) try: save = {'ubernear.last_lookup': now} mongo.save_no_replace( owners_coll, _id=owner_id, save=save, ) except Exception: log.error( 'Could not update owner {_id}'.format( _id=owner_id, ) )
def _save_events( events, events_coll, graph, now, _log=None, ): if _log is None: _log = log # Don't waste a call to the Facebook Graph API if not events: return batch = [ OrderedDict([ ('method', 'GET'), ('relative_url', '{event_id}?date_format=c'.format( event_id=event['_id'] ), ), ]) for event in events ] reponses = graph.batch(batch) for event,response in zip(events,reponses): if isinstance(response, FacepyError): _mark_as_failed( events_coll=events_coll, event_id=event['_id'], now=now, field='lookup_failed', reason=str(response), ) continue # Event does not exist anymore if response is False: _mark_as_failed( events_coll=events_coll, event_id=event['_id'], now=now, field='lookup_failed', reason='False response', ) continue if response is None: # None has special significance in mongodb searches # so use 'null' instead. _mark_as_failed( events_coll=events_coll, event_id=event['_id'], now=now, field='lookup_failed', reason='Null response', ) continue # We seem to have a valid response but ids are different? if response['id'] != event['_id']: _log.error( 'Facebook returned information for an event other than ' '{event_id}. Skipping event.'.format( event_id=event['_id'], ) ) _mark_as_failed( events_coll=events_coll, event_id=event['_id'], now=now, field='lookup_failed', reason='Response id is different', ) continue save = OrderedDict([ ('facebook', response), ('ubernear', OrderedDict([ # Depending on where the event came from, # the event source may not have already # been set ('source', 'facebook'), ('lookup_completed', now), ]), ), ]) # Skip responses without a start_time or end_time. # Sometimes the Graph API returns events without these if ( 'start_time' in save['facebook'] and 'end_time' in save['facebook'] ): save['facebook']['start_time'] = utc_from_iso8601( save['facebook']['start_time'], naive=True, ) save['facebook']['end_time'] = utc_from_iso8601( save['facebook']['end_time'], naive=True, ) else: _mark_as_failed( events_coll=events_coll, event_id=event['_id'], now=now, field='lookup_failed', reason='Missing start_time or end_time', ) continue if 'updated_time' in save['facebook']: save['facebook']['updated_time'] = utc_from_iso8601( save['facebook']['updated_time'], naive=True, ) _log.debug( 'Storing event {event_id}'.format( event_id=event['_id'], ) ) mongo.save_no_replace( events_coll, _id=event['_id'], save=save, )
def test_utc_from_iso_to_naive_utc(): dt = util.utc_from_iso8601( '2011-10-12T19:55:58.345128+0000', naive=True, ) eq(dt, datetime(2011, 10, 12, 19, 55, 58, 345128))
def test_utc_from_iso_to_naive(): dt = util.utc_from_iso8601( '2011-11-16T18:36:06.795119-08:00', naive=True, ) eq(dt, datetime(2011, 11, 17, 2, 36, 06, 795119))
def test_utc_from_iso_aware_utc(): dt = util.utc_from_iso8601('2011-10-12T19:55:58.345128+0000') eq(dt, datetime(2011, 10, 12, 19, 55, 58, 345128, tz.tzutc()))
def test_utc_from_iso_aware(): dt = util.utc_from_iso8601('2011-11-16T18:36:06.795119-08:00') eq(dt, datetime(2011, 11, 17, 2, 36, 06, 795119, tz.tzutc()))