def next_query_time(self, post_count):
if post_count > 5:
return datetimes.now() + datetime.timedelta(
minutes=random.randrange(0, 2))
else:
return datetimes.now() + datetime.timedelta(
minutes=random.randrange(10, 20))
def create_subscription(username, query, **details):
subscription = {
'_id': u'{0}_{1}'.format(username, query),
'username': username,
'query': query,
'datetime': datetimes.now()
}
subscription.update(details)
return subscription
def save_user(username, hashed_password, contact, plan, expiration_date=None):
user = {
'_id': username,
'username': username,
'datetime': datetimes.now(),
'hashed_password': hashed_password,
'contact': contact,
'plan': plan
}
if expiration_date: user['expiration_date'] = expiration_date
with get_collection() as collection:
return collection.insert(user, safe=True)
def disable_expired_trial_users():
spec = {
'expiration_date':{'$lte':datetimes.now()},
'plan.name':'trial',
'$or': [
{'disabled':False},
{'disabled':{'$exists':False}}
]
}
with get_collection() as collection:
usernames = [user['_id'] for user in collection.find(spec)]
if not usernames: return
disable_users(spec)
return usernames
def fetch_historic_platform(query, platform):
subscription = Subscription(query)
manager = SubscriptionManager()
if not subscription:
logger.warning(
u"%s not found in the subscription list. (Maybe have been deleted.)"
% query)
return
if subscription.has_historic_data(platform):
logger.debug(u"Already fetched historic from %s for %s. Skipping..." %
(platform, query))
return
manager.mark_earliest_datetime(query, datetimes.now(),
platform) # lock the subscription
results = active_platforms[platform]().search(
subscription.get_query_obj(),
historic=True,
age_filter=get_age_filter(subscription.get_created_datetime(), True))
logger.info("Pushing results to SQS...")
push(results)
logger.info("Updating earlist_datetime...")
e_datetime = None
for d in [r['datetime'] for r in results]:
if not e_datetime:
e_datetime = d
continue
if e_datetime > d:
e_datetime = d
if e_datetime:
manager.mark_earliest_datetime(query,
datetimes.extract_datetime(e_datetime),
platform)
logger.info(u"Finished fetching historic %s from %s" % (query, platform))
def mark_queried(query):
spec = {'query': query}
updates = {'$set': {'last_queried': datetimes.now()}}
with get_collection() as collection:
return collection.update(spec, updates)
def next_query_time(self, post_count):
return datetimes.now() + datetime.timedelta(
hours=3 * random.randrange(10, 20) / 10)
def format(self, record):
return now() + ' %(levelname)s %(filename)s:%(lineno)d %(funcName)s%(args)s %(msg)s' % record.__dict__