def __init__(self, api_key, servers=TEST):
"""Sets up this module to make API calls. The first argument is the developer's
API key and the second specifies which servers should be used.
Arguments:
api_key -- The developer's API key
servers -- How the server URLs should be set. Must be PRODUCTION or TEST.
"""
self.helper = APIHelper(api_key, servers)
def __init__(self, api_key, servers=TEST):
"""Sets up this module to make API calls. The first argument is the developer's
API key and the second specifies which servers should be used.
Arguments:
api_key -- The developer's API key
servers -- How the server URLs should be set. Must be PRODUCTION or TEST.
"""
self.helper = APIHelper(api_key, servers)
class APIs(object):
urls = {}
def __init__(self, api_key, servers=TEST):
"""Sets up this module to make API calls. The first argument is the developer's
API key and the second specifies which servers should be used.
Arguments:
api_key -- The developer's API key
servers -- How the server URLs should be set. Must be PRODUCTION or TEST.
"""
self.helper = APIHelper(api_key, servers)
#order endpoints
def order_guest(self, rid, em, tray, tip, first_name, last_name, phone, zip, addr, city, state, card_number, card_cvc, card_expiry, card_bill_addr, card_bill_city, card_bill_state, card_bill_zip, card_bill_phone, addr2=None, card_name=None, card_bill_addr2=None, delivery_date=None, delivery_time=None):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
em--The customer's email address
tray--Represents a tray of menu items in the format '[menu item id]/[qty],[option id],...,[option id]'
tip--Tip amount in dollars and cents
first_name--The customer's first name
last_name--The customer's last name
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
city--The city part of the address
state--The state part of the address
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
card_bill_addr--The credit card's billing street address
card_bill_city--The credit card's billing city
card_bill_state--The credit card's billing state
card_bill_zip--The credit card's billing zip code
card_bill_phone--The credit card's billing phone number
Keyword Arguments:
addr2--The second part of the street address, if needed
card_name--Full name as it appears on the credit card
card_bill_addr2--The second part of the credit card's biling street address.
Either
delivery_date--Delivery date
delivery_time--Delivery time
OR
delivery_date--Delivery date"""
return self.helper._call_endpoint('order', 'order_guest', ["rid"], rid=rid, em=em, tray=tray, tip=tip, first_name=first_name, last_name=last_name, phone=phone, zip=zip, addr=addr, city=city, state=state, card_number=card_number, card_cvc=card_cvc, card_expiry=card_expiry, card_bill_addr=card_bill_addr, card_bill_city=card_bill_city, card_bill_state=card_bill_state, card_bill_zip=card_bill_zip, card_bill_phone=card_bill_phone, addr2=addr2, card_name=card_name, card_bill_addr2=card_bill_addr2, delivery_date=delivery_date, delivery_time=delivery_time)
def order_user(self, rid, tray, tip, first_name, last_name, email, current_password, phone=None, zip=None, addr=None, addr2=None, city=None, state=None, nick=None, card_name=None, card_number=None, card_cvc=None, card_expiry=None, card_bill_addr=None, card_bill_addr2=None, card_bill_city=None, card_bill_state=None, card_bill_zip=None, card_bill_phone=None, card_nick=None, delivery_date=None, delivery_time=None):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
tray--Represents a tray of menu items in the format '[menu item id]/[qty],[option id],...,[option id]'
tip--Tip amount in dollars and cents
first_name--The customer's first name
last_name--The customer's last name
email -- The user's email
current_password -- The user's current password
Keyword Arguments:
Either
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
addr2--The second part of the street address, if needed
city--The city part of the address
state--The state part of the address
OR
nick--The delivery location nickname. (From the user's addresses)
Either
card_name--Full name as it appears on the credit card
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
card_bill_addr--The credit card's billing street address
card_bill_addr2--The second part of the credit card's biling street address.
card_bill_city--The credit card's billing city
card_bill_state--The credit card's billing state
card_bill_zip--The credit card's billing zip code
card_bill_phone--The credit card's billing phone number
OR
card_nick--The credit card nickname. (From the user's credit cards)
Either
delivery_date--Delivery date
delivery_time--Delivery time
OR
delivery_date--Delivery date"""
return self.helper._call_endpoint('order', 'order_user', ["rid"], rid=rid, tray=tray, tip=tip, first_name=first_name, last_name=last_name, email=email, current_password=current_password, phone=phone, zip=zip, addr=addr, addr2=addr2, city=city, state=state, nick=nick, card_name=card_name, card_number=card_number, card_cvc=card_cvc, card_expiry=card_expiry, card_bill_addr=card_bill_addr, card_bill_addr2=card_bill_addr2, card_bill_city=card_bill_city, card_bill_state=card_bill_state, card_bill_zip=card_bill_zip, card_bill_phone=card_bill_phone, card_nick=card_nick, delivery_date=delivery_date, delivery_time=delivery_time)
#restaurant endpoints
def delivery_check(self, datetime, rid, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
rid--Ordr.in's unique restaurant identifier for the restaurant.
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant', 'delivery_check', ["rid", "datetime", "zip", "city", "addr"], datetime=datetime, rid=rid, addr=addr, city=city, zip=zip)
def delivery_list(self, datetime, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant', 'delivery_list', ["datetime", "zip", "city", "addr"], datetime=datetime, addr=addr, city=city, zip=zip)
def fee(self, datetime, rid, subtotal, tip, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
rid--Ordr.in's unique restaurant identifier for the restaurant.
subtotal--The cost of all items in the tray in dollars and cents.
tip--The tip in dollars and cents.
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant', 'fee', ["rid", "subtotal", "tip", "datetime", "zip", "city", "addr"], datetime=datetime, rid=rid, subtotal=subtotal, tip=tip, addr=addr, city=city, zip=zip)
def restaurant_details(self, rid):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant', 'restaurant_details', ["rid"], rid=rid)
#user endpoints
def change_password(self, email, password, current_password):
"""Arguments:
email--The user's email address
password--The user's new password
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'change_password', ["email"], email=email, password=password, current_password=current_password)
def create_account(self, email, pw, first_name, last_name):
"""Arguments:
email--The user's email address
pw--The user's password
first_name--The user's first name
last_name--The user's last name
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'create_account', ["email"], email=email, pw=pw, first_name=first_name, last_name=last_name)
def create_addr(self, email, nick, phone, zip, addr, city, state, current_password, addr2=None):
"""Arguments:
email--The user's email address
nick--The nickname of this address
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
city--The city part of the address
state--The state part of the address
current_password -- The user's current password
Keyword Arguments:
addr2--The second part of the street address, if needed
"""
return self.helper._call_endpoint('user', 'create_addr', ["email", "nick"], email=email, nick=nick, phone=phone, zip=zip, addr=addr, city=city, state=state, current_password=current_password, addr2=addr2)
def create_cc(self, email, nick, card_number, card_cvc, card_expiry, bill_addr, bill_city, bill_state, bill_zip, bill_phone, current_password, bill_addr2=None):
"""Arguments:
email--The user's email address
nick--The nickname of this address
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
bill_addr--The credit card's billing street address
bill_city--The credit card's billing city
bill_state--The credit card's billing state
bill_zip--The credit card's billing zip code
bill_phone--The credit card's billing phone number
current_password -- The user's current password
Keyword Arguments:
bill_addr2--The second part of the credit card's biling street address.
"""
return self.helper._call_endpoint('user', 'create_cc', ["email", "nick"], email=email, nick=nick, card_number=card_number, card_cvc=card_cvc, card_expiry=card_expiry, bill_addr=bill_addr, bill_city=bill_city, bill_state=bill_state, bill_zip=bill_zip, bill_phone=bill_phone, current_password=current_password, bill_addr2=bill_addr2)
def delete_addr(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'delete_addr', ["email", "nick"], email=email, nick=nick, current_password=current_password)
def delete_cc(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'delete_cc', ["email", "nick"], email=email, nick=nick, current_password=current_password)
def get_account_info(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_account_info', ["email"], email=email, current_password=current_password)
def get_all_saved_addrs(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_all_saved_addrs', ["email"], email=email, current_password=current_password)
def get_all_saved_ccs(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_all_saved_ccs', ["email"], email=email, current_password=current_password)
def get_order(self, email, oid, current_password):
"""Arguments:
email--The user's email address
oid--Ordr.in's unique order id number.
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_order', ["email", "oid"], email=email, oid=oid, current_password=current_password)
def get_order_history(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_order_history', ["email"], email=email, current_password=current_password)
def get_saved_addr(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_saved_addr', ["email", "nick"], email=email, nick=nick, current_password=current_password)
def get_saved_cc(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user', 'get_saved_cc', ["email", "nick"], email=email, nick=nick, current_password=current_password)
class APIs(object):
urls = {}
def __init__(self, api_key, servers=TEST):
"""Sets up this module to make API calls. The first argument is the developer's
API key and the second specifies which servers should be used.
Arguments:
api_key -- The developer's API key
servers -- How the server URLs should be set. Must be PRODUCTION or TEST.
"""
self.helper = APIHelper(api_key, servers)
#order endpoints
def order_guest(self,
rid,
em,
tray,
tip,
first_name,
last_name,
phone,
zip,
addr,
city,
state,
card_number,
card_cvc,
card_expiry,
card_bill_addr,
card_bill_city,
card_bill_state,
card_bill_zip,
card_bill_phone,
addr2=None,
card_name=None,
card_bill_addr2=None,
delivery_date=None,
delivery_time=None):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
em--The customer's email address
tray--Represents a tray of menu items in the format '[menu item id]/[qty],[option id],...,[option id]'
tip--Tip amount in dollars and cents
first_name--The customer's first name
last_name--The customer's last name
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
city--The city part of the address
state--The state part of the address
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
card_bill_addr--The credit card's billing street address
card_bill_city--The credit card's billing city
card_bill_state--The credit card's billing state
card_bill_zip--The credit card's billing zip code
card_bill_phone--The credit card's billing phone number
Keyword Arguments:
addr2--The second part of the street address, if needed
card_name--Full name as it appears on the credit card
card_bill_addr2--The second part of the credit card's biling street address.
Either
delivery_date--Delivery date
delivery_time--Delivery time
OR
delivery_date--Delivery date"""
return self.helper._call_endpoint('order',
'order_guest', ["rid"],
rid=rid,
em=em,
tray=tray,
tip=tip,
first_name=first_name,
last_name=last_name,
phone=phone,
zip=zip,
addr=addr,
city=city,
state=state,
card_number=card_number,
card_cvc=card_cvc,
card_expiry=card_expiry,
card_bill_addr=card_bill_addr,
card_bill_city=card_bill_city,
card_bill_state=card_bill_state,
card_bill_zip=card_bill_zip,
card_bill_phone=card_bill_phone,
addr2=addr2,
card_name=card_name,
card_bill_addr2=card_bill_addr2,
delivery_date=delivery_date,
delivery_time=delivery_time)
def order_user(self,
rid,
tray,
tip,
first_name,
last_name,
email,
current_password,
phone=None,
zip=None,
addr=None,
addr2=None,
city=None,
state=None,
nick=None,
card_name=None,
card_number=None,
card_cvc=None,
card_expiry=None,
card_bill_addr=None,
card_bill_addr2=None,
card_bill_city=None,
card_bill_state=None,
card_bill_zip=None,
card_bill_phone=None,
card_nick=None,
delivery_date=None,
delivery_time=None):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
tray--Represents a tray of menu items in the format '[menu item id]/[qty],[option id],...,[option id]'
tip--Tip amount in dollars and cents
first_name--The customer's first name
last_name--The customer's last name
email -- The user's email
current_password -- The user's current password
Keyword Arguments:
Either
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
addr2--The second part of the street address, if needed
city--The city part of the address
state--The state part of the address
OR
nick--The delivery location nickname. (From the user's addresses)
Either
card_name--Full name as it appears on the credit card
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
card_bill_addr--The credit card's billing street address
card_bill_addr2--The second part of the credit card's biling street address.
card_bill_city--The credit card's billing city
card_bill_state--The credit card's billing state
card_bill_zip--The credit card's billing zip code
card_bill_phone--The credit card's billing phone number
OR
card_nick--The credit card nickname. (From the user's credit cards)
Either
delivery_date--Delivery date
delivery_time--Delivery time
OR
delivery_date--Delivery date"""
return self.helper._call_endpoint('order',
'order_user', ["rid"],
rid=rid,
tray=tray,
tip=tip,
first_name=first_name,
last_name=last_name,
email=email,
current_password=current_password,
phone=phone,
zip=zip,
addr=addr,
addr2=addr2,
city=city,
state=state,
nick=nick,
card_name=card_name,
card_number=card_number,
card_cvc=card_cvc,
card_expiry=card_expiry,
card_bill_addr=card_bill_addr,
card_bill_addr2=card_bill_addr2,
card_bill_city=card_bill_city,
card_bill_state=card_bill_state,
card_bill_zip=card_bill_zip,
card_bill_phone=card_bill_phone,
card_nick=card_nick,
delivery_date=delivery_date,
delivery_time=delivery_time)
#restaurant endpoints
def delivery_check(self, datetime, rid, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
rid--Ordr.in's unique restaurant identifier for the restaurant.
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint(
'restaurant',
'delivery_check', ["rid", "datetime", "zip", "city", "addr"],
datetime=datetime,
rid=rid,
addr=addr,
city=city,
zip=zip)
def delivery_list(self, datetime, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant',
'delivery_list',
["datetime", "zip", "city", "addr"],
datetime=datetime,
addr=addr,
city=city,
zip=zip)
def fee(self, datetime, rid, subtotal, tip, addr, city, zip):
"""Arguments:
datetime--Delivery date and time
rid--Ordr.in's unique restaurant identifier for the restaurant.
subtotal--The cost of all items in the tray in dollars and cents.
tip--The tip in dollars and cents.
addr--Delivery location street address
city--Delivery location city
zip--The zip code part of the address
Keyword Arguments:
"""
return self.helper._call_endpoint(
'restaurant',
'fee',
["rid", "subtotal", "tip", "datetime", "zip", "city", "addr"],
datetime=datetime,
rid=rid,
subtotal=subtotal,
tip=tip,
addr=addr,
city=city,
zip=zip)
def restaurant_details(self, rid):
"""Arguments:
rid--Ordr.in's unique restaurant identifier for the restaurant.
Keyword Arguments:
"""
return self.helper._call_endpoint('restaurant',
'restaurant_details', ["rid"],
rid=rid)
#user endpoints
def change_password(self, email, password, current_password):
"""Arguments:
email--The user's email address
password--The user's new password
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'change_password', ["email"],
email=email,
password=password,
current_password=current_password)
def create_account(self, email, pw, first_name, last_name):
"""Arguments:
email--The user's email address
pw--The user's password
first_name--The user's first name
last_name--The user's last name
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'create_account', ["email"],
email=email,
pw=pw,
first_name=first_name,
last_name=last_name)
def create_addr(self,
email,
nick,
phone,
zip,
addr,
city,
state,
current_password,
addr2=None):
"""Arguments:
email--The user's email address
nick--The nickname of this address
phone--The customer's phone number
zip--The zip code part of the address
addr--The street address
city--The city part of the address
state--The state part of the address
current_password -- The user's current password
Keyword Arguments:
addr2--The second part of the street address, if needed
"""
return self.helper._call_endpoint('user',
'create_addr', ["email", "nick"],
email=email,
nick=nick,
phone=phone,
zip=zip,
addr=addr,
city=city,
state=state,
current_password=current_password,
addr2=addr2)
def create_cc(self,
email,
nick,
card_number,
card_cvc,
card_expiry,
bill_addr,
bill_city,
bill_state,
bill_zip,
bill_phone,
current_password,
bill_addr2=None):
"""Arguments:
email--The user's email address
nick--The nickname of this address
card_number--Credit card number
card_cvc--3 or 4 digit security code
card_expiry--The credit card expiration date.
bill_addr--The credit card's billing street address
bill_city--The credit card's billing city
bill_state--The credit card's billing state
bill_zip--The credit card's billing zip code
bill_phone--The credit card's billing phone number
current_password -- The user's current password
Keyword Arguments:
bill_addr2--The second part of the credit card's biling street address.
"""
return self.helper._call_endpoint('user',
'create_cc', ["email", "nick"],
email=email,
nick=nick,
card_number=card_number,
card_cvc=card_cvc,
card_expiry=card_expiry,
bill_addr=bill_addr,
bill_city=bill_city,
bill_state=bill_state,
bill_zip=bill_zip,
bill_phone=bill_phone,
current_password=current_password,
bill_addr2=bill_addr2)
def delete_addr(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'delete_addr', ["email", "nick"],
email=email,
nick=nick,
current_password=current_password)
def delete_cc(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'delete_cc', ["email", "nick"],
email=email,
nick=nick,
current_password=current_password)
def get_account_info(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_account_info', ["email"],
email=email,
current_password=current_password)
def get_all_saved_addrs(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_all_saved_addrs', ["email"],
email=email,
current_password=current_password)
def get_all_saved_ccs(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_all_saved_ccs', ["email"],
email=email,
current_password=current_password)
def get_order(self, email, oid, current_password):
"""Arguments:
email--The user's email address
oid--Ordr.in's unique order id number.
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_order', ["email", "oid"],
email=email,
oid=oid,
current_password=current_password)
def get_order_history(self, email, current_password):
"""Arguments:
email--The user's email address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_order_history', ["email"],
email=email,
current_password=current_password)
def get_saved_addr(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_saved_addr', ["email", "nick"],
email=email,
nick=nick,
current_password=current_password)
def get_saved_cc(self, email, nick, current_password):
"""Arguments:
email--The user's email address
nick--The nickname of this address
current_password -- The user's current password
Keyword Arguments:
"""
return self.helper._call_endpoint('user',
'get_saved_cc', ["email", "nick"],
email=email,
nick=nick,
current_password=current_password)
def __init__(self):
self.api = APIHelper()
self.data = DataManager()
self.web = WebHelper()
class Fetcher(object):
"""API fetcher to complete tasks"""
def __init__(self):
self.api = APIHelper()
self.data = DataManager()
self.web = WebHelper()
def fetch_single_topic(self, topic_id):
topic_info = self.api.get_topic_info(topic_id)
partial_member = self.data.member_of_topic(topic_info)
stored_topic = self.data.find_topic(topic_id)
new_topic = self.data.handle_topic(topic_info, topic_id)
if stored_topic:
new_topic['web_crawled'] = stored_topic['web_crawled']
else:
new_topic['web_crawled'] = datetime.fromtimestamp(0)
new_topic['click'] = 0
new_topic['favorite'] = 0
new_topic['thank'] = 0
self.data.update_topic_synced_state(new_topic['id'], False)
self.data.upsert_topic(new_topic)
self.data.upsert_member(partial_member)
def fetch_new_topics(self):
max_stored_topic_id = self.data.max_stored_topic_id
topic_count = self.api.get_topic_count()
if max_stored_topic_id >= topic_count:
return
for topic_id in range(max_stored_topic_id + 1, topic_count + 1):
self.fetch_single_topic(topic_id)
def fetch_replies_of_topic(self, topic_id):
replies = self.api.get_replies(topic_id)
if replies:
self.data.update_topic_synced_state(topic_id, False)
for reply in replies:
partial_member = self.data.member_of_reply(reply)
self.data.upsert_reply(self.data.handle_reply(reply, topic_id))
self.data.upsert_member(partial_member)
def fetch_new_replies(self):
max_stored_topic_id = self.data.max_stored_topic_id
max_stored_topic_id_of_reply = self.data.max_stored_topic_id_of_reply
if max_stored_topic_id_of_reply == 0:
need_refetch_max_topic = False
else:
topic = self.data.find_topic(max_stored_topic_id_of_reply)
replies = self.data.find_all_replies(max_stored_topic_id_of_reply)
need_refetch_max_topic = not (replies.count() == topic['replies'])
for topic_id in range(
max_stored_topic_id_of_reply + 1 - int(need_refetch_max_topic),
max_stored_topic_id + 1):
if self.data.find_topic(topic_id=topic_id):
self.fetch_replies_of_topic(topic_id)
def fetch_single_topic_extras(self, topic_id):
def upsert_counts(click, favorite, thank):
self.data.upsert_topic_extras(topic_id, click, favorite, thank)
self.data.update_topic_synced_state(topic_id, False)
count = self.data.update_topic_web_crawled(topic_id,
datetime.utcnow())
logging.info('Update topic {0} extras, count {1}'.format(
topic_id, count))
web_extra = self.web.get_topic_extras(topic_id)
if not web_extra:
count = self.data.update_topic_web_crawled(topic_id,
datetime.utcnow())
topic = self.data.find_topic(topic_id)
if (not topic) or ('click' not in topic):
upsert_counts(0, 0, 0)
logging.info('Topic {0} extras is None, count {1}'.format(
topic_id, count))
return
for index, postscript in enumerate(web_extra.subtle_list):
postscript = self.data.handle_postscript(postscript, topic_id,
index + 1)
self.data.upsert_postscript(postscript)
upsert_counts(web_extra.click, web_extra.favorite, web_extra.thank)
def fetch_all_topic_extras(self):
while True:
min_topic_id_need_postscript = self.data.min_topic_id_need_extras
if min_topic_id_need_postscript > 0:
try:
self.fetch_single_topic_extras(
min_topic_id_need_postscript)
except ElasticsearchException as es_error:
raise es_error
except Exception as e:
logging.error('Fetch single topic extras error: ' + str(e))
else:
break
def fetch_all_nodes(self):
nodes = self.api.get_all_nodes()
if nodes:
for node in nodes:
node['crawled'] = datetime.utcnow()
self.data.upsert_node(node)
logging.info('Fetching all node, count: ' + str(len(nodes)))
def fetch_new_members(self):
site_stats = self.api.get_site_stats()
max_stored_member_id = self.data.max_stored_member_id
if site_stats:
member_max = site_stats['member_max']
for member_id in range(max_stored_member_id + 1, member_max + 1):
new_member = self.api.get_member_info(member_id=member_id)
if not new_member:
continue
new_member['crawled'] = datetime.utcnow()
self.data.upsert_member(new_member)
logging.info('Upsert member {0}, id {1}'.format(
new_member['username'], new_member['id']))
else:
new_member = self.api.get_member_info(
member_id=self.data.max_stored_member_id + 1)
while new_member:
new_member['crawled'] = datetime.utcnow()
self.data.upsert_member(new_member)
logging.info('Upsert member {0}, id {1}'.format(
new_member['username'], new_member['id']))
new_member = self.api.get_member_info(
member_id=self.data.max_stored_member_id + 1)
def fetch_stale_topics(self):
stale_topics = self.data.stale_topics()
if not stale_topics:
return
for topic in stale_topics:
topic_id = topic['id']
self.fetch_single_topic(topic_id)
if topic['web_crawled'] and topic[
'recrawl'] and topic['web_crawled'] < topic['recrawl']:
self.fetch_single_topic_extras(topic_id)
self.fetch_replies_of_topic(topic_id)
def sync_topic_to_es(self):
waiting_topics = self.data.not_synced_topics()
if not waiting_topics:
return
for topic in waiting_topics:
self.data.update_topic_synced_state(topic['id'], True)
self.data.es_update_assembled_topic(topic)
})
config.logger.debug("Created config collection")
config.logger.debug("\thash tags: {}".format(config.HASH_TAGS))
config.logger.debug("\tnum tweets to search: {}".format(
config.NUM_TWEETS_TO_SEARCH))
config.logger.debug("\tpull tweets interval: {}".format(
config.PULL_TWEETS_INTERVAL))
for hash_tag in config.HASH_TAGS:
config.logger.debug("\t\thash_tag: {}".format(hash_tag))
def run(tw_api, mongo_client):
while True:
config.logger.info("App is running")
initialize(mongo_client)
if not config.PAUSE_APP:
pull_tweets(tw_api, mongo_client)
process_pulled_tweets(tw_api, mongo_client)
time.sleep(30)
if __name__ == "__main__":
api_helper = APIHelper()
tw_api, mongo_client = api_helper.create_api()
config.logger.info("App started")
run(tw_api, mongo_client)
