def main():
parser = optparse.OptionParser(
usage='%prog [OPTS]',
)
parser.add_option(
'-v', '--verbose',
help='Verbose mode [default %default]',
action="store_true", dest="verbose"
)
parser.add_option(
'--config',
help=('Path to the file with information on how to '
'configure facebook-event'
),
metavar='PATH',
)
parser.add_option(
'--db-config',
help=('Path to the file with information on how to '
'retrieve and store data in the database'
),
metavar='PATH',
)
parser.add_option(
'-a', '--process-all',
help=('Process all events that have not expired '
'instead of just those that have not been '
'processed [default %default]'
),
action="store_true", dest="process_all"
)
parser.set_defaults(
verbose=False,
process_all=False,
)
options, args = parser.parse_args()
if args:
parser.error('Wrong number of arguments.')
if options.config is None:
parser.error('Missing option --config=.')
if options.db_config is None:
parser.error('Missing option --db-config=.')
logging.basicConfig(
level=logging.DEBUG if options.verbose else logging.INFO,
format='%(asctime)s.%(msecs)03d %(name)s: %(levelname)s: %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
config = config_parser(options.config)
access_token = config.get('facebook', 'access_token')
graph = GraphAPI(access_token)
usps_id = config.get('usps', 'user_id')
yahoo_id = config.get('yahoo', 'app_id')
coll = collections(options.db_config)
events_coll = coll['events-collection']
expired_coll = coll['expired-collection']
indices = [
{'facebook.end_time': pymongo.ASCENDING},
{'ubernear.fetched': pymongo.ASCENDING},
]
mongo.create_indices(
collection=events_coll,
indices=indices,
)
indices = [
{'facebook.end_time': pymongo.ASCENDING},
]
mongo.create_indices(
collection=expired_coll,
indices=indices,
)
log.info('Start...')
log.info('Moving expired events...')
facebook_event.expire(
events_coll=events_coll,
expired_coll=expired_coll,
)
log.info('Updating event data...')
facebook_work = facebook_event.update_facebook(
events_coll=events_coll,
graph=graph,
process_all=options.process_all,
)
log.info('Updating venue data...')
venue_work = facebook_event.update_venue(
events_coll=events_coll,
usps_id=usps_id,
process_all=options.process_all,
)
log.info('Updating coordinate data...')
coord_work = facebook_event.update_coordinate(
events_coll=events_coll,
yahoo_id=yahoo_id,
process_all=options.process_all,
)
if coord_work['sleep'] is not None:
delay = coord_work['sleep']
log.info(
'Geocoding rate limit reached. '
'Sleeping {sleep} hours...'.format(
delay=delay,
)
)
time.sleep(delay)
else:
found_work = (
facebook_work
or
venue_work
or
coord_work['found_work']
)
if not found_work:
hours = 24
delay = random.randint(60*60*hours, 60*60*hours+1)
log.info(
'Did not find any work. '
'Sleeping about {hours} hours...'.format(
hours=hours,
)
)
time.sleep(delay)
log.info('End')
def test_expire_place_ids(self):
events_coll = fudge.Fake("events_coll")
events_coll.remember_order()
expired_coll = fudge.Fake("expired_coll")
expired_coll.remember_order()
find = events_coll.expects("find")
last_week = datetime(2012, 3, 10, 7, 23, 19)
end_parts = [
OrderedDict([("facebook.end_time", OrderedDict([("$exists", True)]))]),
OrderedDict([("facebook.end_time", OrderedDict([("$lt", last_week)]))]),
]
end_query = OrderedDict([("$and", end_parts)])
false_query = OrderedDict([("ubernear.lookup_failed.reason", "False response")])
unsupported_query = OrderedDict(
[
(
"ubernear.lookup_failed.reason",
OrderedDict(
[
("$regex", "GraphMethodException error on get.*" ": Unsupported get request.."),
("$options", "i"),
]
),
)
]
)
alias_query = OrderedDict(
[
(
"ubernear.lookup_failed.reason",
OrderedDict(
[
(
"$regex",
"OAuthException error on get.*Some " "of the aliases you requested do not exist.*",
),
("$options", "i"),
]
),
)
]
)
or_query = OrderedDict([("$or", [false_query, unsupported_query, alias_query])])
facebook_query = OrderedDict([("ubernear.lookup_completed", OrderedDict([("$exists", False)]))])
failed_query = OrderedDict([("$and", [facebook_query, or_query])])
find.with_args(OrderedDict([("$or", [end_query, failed_query])]), sort=[("facebook.end_time", 1)])
ubernear_1 = OrderedDict([("place_ids", ["bd2541a8-6a99-4da1-b82c-547bbebf6ee7"])])
ubernear_2 = OrderedDict([("place_ids", ["9807155b-1cba-4a34-8d0e-40e70dffcf8d"])])
facebook_1 = OrderedDict(
[
("title", "event title 226680217397995"),
("id", "226680217397995"),
("start_time", datetime(2012, 2, 26, 8, 0, 0)),
("end_time", datetime(2012, 2, 26, 11, 0, 0)),
("updated_time", datetime(2011, 10, 27, 22, 56, 42)),
]
)
facebook_2 = OrderedDict(
[
("title", "event title 267558763278075"),
("id", "267558763278075"),
("start_time", datetime(2011, 10, 28, 22, 0, 0)),
("end_time", datetime(2011, 10, 29, 4, 0, 0)),
("updated_time", datetime(2011, 10, 12, 19, 55, 58)),
]
)
find.returns(
[
OrderedDict([("_id", "226680217397995"), ("facebook", facebook_1), ("ubernear", ubernear_1)]),
OrderedDict([("_id", "267558763278075"), ("facebook", facebook_2), ("ubernear", ubernear_2)]),
]
)
update = expired_coll.expects("update")
save = OrderedDict(
[
("facebook.title", "event title 226680217397995"),
("facebook.id", "226680217397995"),
("facebook.start_time", datetime(2012, 2, 26, 8, 0, 0)),
("facebook.end_time", datetime(2012, 2, 26, 11, 0, 0)),
("facebook.updated_time", datetime(2011, 10, 27, 22, 56, 42)),
]
)
add_each = OrderedDict(
[("ubernear.place_ids", OrderedDict([("$each", ["bd2541a8-6a99-4da1-b82c-547bbebf6ee7"])]))]
)
update.with_args(
OrderedDict([("_id", "226680217397995")]),
OrderedDict([("$set", save), ("$addToSet", add_each)]),
upsert=True,
safe=True,
)
update = expired_coll.next_call("update")
save = OrderedDict(
[
("facebook.title", "event title 267558763278075"),
("facebook.id", "267558763278075"),
("facebook.start_time", datetime(2011, 10, 28, 22, 0, 0)),
("facebook.end_time", datetime(2011, 10, 29, 4, 0, 0)),
("facebook.updated_time", datetime(2011, 10, 12, 19, 55, 58)),
]
)
add_each = OrderedDict(
[("ubernear.place_ids", OrderedDict([("$each", ["9807155b-1cba-4a34-8d0e-40e70dffcf8d"])]))]
)
update.with_args(
OrderedDict([("_id", "267558763278075")]),
OrderedDict([("$set", save), ("$addToSet", add_each)]),
upsert=True,
safe=True,
)
remove = events_coll.expects("remove")
remove.with_args(OrderedDict([("_id", "226680217397995")]))
remove = events_coll.next_call("remove")
remove.with_args(OrderedDict([("_id", "267558763278075")]))
fake_datetime = fudge.Fake("datetime")
fake_datetime.remember_order()
utcnow = fake_datetime.expects("utcnow")
utcnow.with_arg_count(0)
utcnow.returns(datetime(2012, 3, 17, 7, 23, 19))
facebook_event.expire(events_coll=events_coll, expired_coll=expired_coll, _datetime=fake_datetime)