def test_update_facebook_no_work(self): events_coll = fudge.Fake("events_coll") events_coll.remember_order() find = events_coll.expects("find") null_query = OrderedDict([("ubernear.lookup_failed.reason", "Null response")]) validating_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict([("$regex", "OAuthException error on get.*: " "Error validating application..")]), ) ] ) retry_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict( [ ( "$regex", "OAuthException error on get.*: " "An unexpected error has occurred. " "Please retry your request later..", ) ] ), ) ] ) or_query = OrderedDict([("$or", [null_query, validating_query, retry_query])]) match_query = OrderedDict([("ubernear.lookup_completed", OrderedDict([("$exists", False)]))]) and_query = OrderedDict([("$and", [match_query, or_query])]) query = OrderedDict([("$or", [match_query, and_query])]) find.with_args(query, sort=[("ubernear.fetched", 1)]) fake_cursor = FakeCursor([]) find.returns(fake_cursor) fake_datetime = fudge.Fake("datetime") fake_datetime.remember_order() utcnow = fake_datetime.expects("utcnow") utcnow.with_arg_count(0) utcnow.returns(datetime(2011, 10, 16, 2, 50, 32)) fake_graph = fudge.Fake("graph") fake_log = fudge.Fake("log") found_work = facebook_event.update_facebook( events_coll=events_coll, graph=fake_graph, _log=fake_log, _datetime=fake_datetime ) eq(found_work, False)
def test_update_facebook_process_all(self): events_coll = fudge.Fake("events_coll") events_coll.remember_order() find = events_coll.expects("find") find.with_arg_count(0) ubernear_1 = OrderedDict([("fetched", datetime(2011, 11, 14, 1, 15, 53))]) ubernear_2 = OrderedDict([("fetched", datetime(2011, 11, 15, 2, 36, 21))]) events = [ OrderedDict([("_id", "226680217397995"), ("ubernear", ubernear_1)]), OrderedDict([("_id", "267558763278075"), ("ubernear", ubernear_2)]), ] fake_cursor = FakeCursor(events) find.returns(fake_cursor) fake_log = fudge.Fake("log") fake_log.remember_order() info = fake_log.expects("info") info.with_args("Fetching 2 events") self._create_save_events_simple_fakes(events_coll=events_coll, fake_log=fake_log) fake_datetime = fudge.Fake("datetime") fake_datetime.remember_order() utcnow = fake_datetime.expects("utcnow") utcnow.with_arg_count(0) utcnow.returns(datetime(2011, 11, 16, 2, 50, 32)) found_work = facebook_event.update_facebook( events_coll=self._events_coll, graph=self._fake_graph, process_all=True, _log=self._fake_log, _datetime=fake_datetime, ) eq(found_work, True)
def main(): parser = optparse.OptionParser( usage='%prog [OPTS]', ) parser.add_option( '-v', '--verbose', help='Verbose mode [default %default]', action="store_true", dest="verbose" ) parser.add_option( '--config', help=('Path to the file with information on how to ' 'configure facebook-event' ), metavar='PATH', ) parser.add_option( '--db-config', help=('Path to the file with information on how to ' 'retrieve and store data in the database' ), metavar='PATH', ) parser.add_option( '-a', '--process-all', help=('Process all events that have not expired ' 'instead of just those that have not been ' 'processed [default %default]' ), action="store_true", dest="process_all" ) parser.set_defaults( verbose=False, process_all=False, ) options, args = parser.parse_args() if args: parser.error('Wrong number of arguments.') if options.config is None: parser.error('Missing option --config=.') if options.db_config is None: parser.error('Missing option --db-config=.') logging.basicConfig( level=logging.DEBUG if options.verbose else logging.INFO, format='%(asctime)s.%(msecs)03d %(name)s: %(levelname)s: %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) config = config_parser(options.config) access_token = config.get('facebook', 'access_token') graph = GraphAPI(access_token) usps_id = config.get('usps', 'user_id') yahoo_id = config.get('yahoo', 'app_id') coll = collections(options.db_config) events_coll = coll['events-collection'] expired_coll = coll['expired-collection'] indices = [ {'facebook.end_time': pymongo.ASCENDING}, {'ubernear.fetched': pymongo.ASCENDING}, ] mongo.create_indices( collection=events_coll, indices=indices, ) indices = [ {'facebook.end_time': pymongo.ASCENDING}, ] mongo.create_indices( collection=expired_coll, indices=indices, ) log.info('Start...') log.info('Moving expired events...') facebook_event.expire( events_coll=events_coll, expired_coll=expired_coll, ) log.info('Updating event data...') facebook_work = facebook_event.update_facebook( events_coll=events_coll, graph=graph, process_all=options.process_all, ) log.info('Updating venue data...') venue_work = facebook_event.update_venue( events_coll=events_coll, usps_id=usps_id, process_all=options.process_all, ) log.info('Updating coordinate data...') coord_work = facebook_event.update_coordinate( events_coll=events_coll, yahoo_id=yahoo_id, process_all=options.process_all, ) if coord_work['sleep'] is not None: delay = coord_work['sleep'] log.info( 'Geocoding rate limit reached. ' 'Sleeping {sleep} hours...'.format( delay=delay, ) ) time.sleep(delay) else: found_work = ( facebook_work or venue_work or coord_work['found_work'] ) if not found_work: hours = 24 delay = random.randint(60*60*hours, 60*60*hours+1) log.info( 'Did not find any work. ' 'Sleeping about {hours} hours...'.format( hours=hours, ) ) time.sleep(delay) log.info('End')
def test_update_facebook_more_than_50(self): fake_graph = fudge.Fake("graph") fake_graph.remember_order() def batch_fakes(fake_graph, start, end): if start == 1: batch = fake_graph.expects("batch") else: batch = fake_graph.next_call("batch") request = [ OrderedDict([("method", "GET"), ("relative_url", "{i}?date_format=c".format(i=i))]) for i in xrange(start, end) ] batch.with_args(request) batch_response = [] for i in xrange(start, end): min_ = 15 sec = i if sec > 59: min_ += sec / 59 sec %= 59 response = OrderedDict( [ ("title", "event title {i}".format(i=i)), ("id", "{i}".format(i=i)), ("start_time", "2012-02-26T08:{:02}:{:02}+00:00".format(min_, sec)), ("end_time", "2012-02-26T11:{:02}:{:02}+00:00".format(min_, sec)), ] ) batch_response.append(response) batch.returns(batch_response) batch_fakes(fake_graph, 1, 50 + 1) batch_fakes(fake_graph, 51, 100 + 1) batch_fakes(fake_graph, 101, 105 + 1) events_coll = fudge.Fake("events_coll") events_coll.remember_order() find = events_coll.expects("find") null_query = OrderedDict([("ubernear.lookup_failed.reason", "Null response")]) validating_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict([("$regex", "OAuthException error on get.*: " "Error validating application..")]), ) ] ) retry_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict( [ ( "$regex", "OAuthException error on get.*: " "An unexpected error has occurred. " "Please retry your request later..", ) ] ), ) ] ) or_query = OrderedDict([("$or", [null_query, validating_query, retry_query])]) match_query = OrderedDict([("ubernear.lookup_completed", OrderedDict([("$exists", False)]))]) and_query = OrderedDict([("$and", [match_query, or_query])]) query = OrderedDict([("$or", [match_query, and_query])]) find.with_args(query, sort=[("ubernear.fetched", 1)]) events = [] for i in xrange(1, 105 + 1): min_ = 15 sec = i if sec > 59: min_ += sec / 59 sec %= 59 ubernear = OrderedDict([("fetched", datetime(2011, 11, 14, 1, min_, sec))]) events.append(OrderedDict([("_id", "{i}".format(i=i)), ("ubernear", ubernear)])) fake_cursor = FakeCursor(events) find.returns(fake_cursor) fake_log = fudge.Fake("log") fake_log.remember_order() info = fake_log.expects("info") info.with_args("Fetching 105 events") def debug_fakes(fake_log, i): if i == 1: debug = fake_log.expects("debug") else: debug = fake_log.next_call("debug") debug.with_args("Storing event {i}".format(i=i)) [debug_fakes(fake_log, i) for i in xrange(1, 105 + 1)] def update_fakes(events_coll, i): if i == 1: update = events_coll.expects("update") else: update = events_coll.next_call("update") min_ = 15 sec = i if sec > 59: min_ += sec / 59 sec %= 59 save = OrderedDict( [ ("facebook.title", "event title {i}".format(i=i)), ("facebook.id", "{i}".format(i=i)), ("facebook.start_time", datetime(2012, 2, 26, 8, min_, sec)), ("facebook.end_time", datetime(2012, 2, 26, 11, min_, sec)), ("ubernear.source", "facebook"), ("ubernear.lookup_completed", datetime(2011, 10, 16, 2, 50, 32)), ] ) update.with_args( OrderedDict([("_id", "{i}".format(i=i))]), OrderedDict([("$set", save)]), upsert=True, safe=True ) [update_fakes(events_coll, i) for i in xrange(1, 105 + 1)] fake_datetime = fudge.Fake("datetime") fake_datetime.remember_order() utcnow = fake_datetime.expects("utcnow") utcnow.with_arg_count(0) utcnow.returns(datetime(2011, 10, 16, 2, 50, 32)) found_work = facebook_event.update_facebook( events_coll=events_coll, graph=fake_graph, _log=fake_log, _datetime=fake_datetime ) eq(found_work, True)
def test_update_facebook_simple(self): events_coll = fudge.Fake("events_coll") events_coll.remember_order() find = events_coll.expects("find") null_query = OrderedDict([("ubernear.lookup_failed.reason", "Null response")]) validating_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict([("$regex", "OAuthException error on get.*: " "Error validating application..")]), ) ] ) retry_query = OrderedDict( [ ( "ubernear.lookup_failed.reason", OrderedDict( [ ( "$regex", "OAuthException error on get.*: " "An unexpected error has occurred. " "Please retry your request later..", ) ] ), ) ] ) or_query = OrderedDict([("$or", [null_query, validating_query, retry_query])]) match_query = OrderedDict([("ubernear.lookup_completed", OrderedDict([("$exists", False)]))]) and_query = OrderedDict([("$and", [match_query, or_query])]) query = OrderedDict([("$or", [match_query, and_query])]) find.with_args(query, sort=[("ubernear.fetched", 1)]) ubernear_1 = OrderedDict([("fetched", datetime(2011, 11, 14, 1, 15, 53))]) ubernear_2 = OrderedDict([("fetched", datetime(2011, 11, 15, 2, 36, 21))]) events = [ OrderedDict([("_id", "226680217397995"), ("ubernear", ubernear_1)]), OrderedDict([("_id", "267558763278075"), ("ubernear", ubernear_2)]), ] fake_cursor = FakeCursor(events) find.returns(fake_cursor) fake_log = fudge.Fake("log") fake_log.remember_order() info = fake_log.expects("info") info.with_args("Fetching 2 events") self._create_save_events_simple_fakes(events_coll=events_coll, fake_log=fake_log) fake_datetime = fudge.Fake("datetime") fake_datetime.remember_order() utcnow = fake_datetime.expects("utcnow") utcnow.with_arg_count(0) utcnow.returns(datetime(2011, 11, 16, 2, 50, 32)) found_work = facebook_event.update_facebook( events_coll=self._events_coll, graph=self._fake_graph, _log=self._fake_log, _datetime=fake_datetime ) eq(found_work, True)