def fetch_tweet_info(url, headers = {'accept': 'application/json'}):
logger.debug("Fetching {} URL".format(url))
headers, response_json = make_api_request(url=url, method='GET', headers=headers)
global g_headers
g_headers = headers
if isinstance(response_json, dict) and ('errors' in response_json.keys() or 'error' in response_json.keys()):
if 'errors' in response_json.keys():
errors = response_json['errors']
else:
errors = [{"message":response_json['error']}]
logger.error("Encountered error {} while accessing {} URL".format(response_json, url))
for error in errors:
if 'code' in error.keys():
if error['code'] == 88:
raise TwitterRateLimitError(response_json)
elif error['code'] == 50:
raise TwitterUserNotFoundError(response_json)
elif error['code'] == 89:
raise TwitterUserInvalidOrExpiredToken(response_json)
elif error['code'] == 326:
raise TwitterUserAccountLocked(response_json)
elif error['code'] == 34:
raise TwitterPageDoesnotExist(response_json)
else:
raise TwitterUnknownError(response_json)
else:
raise TwitterUnknownError(response_json)
raise TwitterUnknownError(response_json)
return response_json
def get_all_users_list(self):
print("Finding users from DB")
query = """
MATCH (u:User) return u.screen_name ORDER BY u.screen_name
"""
response_json = execute_query_with_result(query)
users = [user['u.screen_name'] for user in response_json]
logger.debug("Got {} users".format(len(users)))
return users
def get_all_users_list(self, in_file='data/twitter_all_users_name.json'):
print("Finding users from file")
response_json = []
try:
with open(in_file) as f:
r = f.read()
if r:
ru = r.encode('utf-8')
decoded_data = codecs.decode(ru, 'utf-8-sig')
response_json = json.loads(decoded_data)
except IOError as e:
print("I/O error({0}): {1}".format(e.errno, e.strerror))
logger.error("File {} I/O error({}): {}".format(
in_file, e.errno, e.strerror))
users = [user['u.screen_name'] for user in response_json]
logger.debug("Got {} users".format(len(users)))
return users
def read_out_file_data(self):
print("Reading Store file data")
in_file = self.outfile
json_data = []
try:
with open(in_file) as f:
json_data = [json.loads(line) for line in f]
except IOError as e:
print("I/O error({0}): {1}".format(e.errno, e.strerror))
logger.error("File {} I/O error({}): {}".format(
in_file, e.errno, e.strerror))
return json_data
users = [
user['target_screen_name'] for user in json_data
if 'can_dm' in user and user['can_dm'] == 1
]
logger.debug("Got {} DM users".format(len(users)))
return users
def get_nonexists_users_list(self):
print("Finding Non existing users from file")
in_file = self.outfile
json_data = []
try:
with open(in_file) as f:
json_data = [json.loads(line) for line in f]
except IOError as e:
print("Info: Couldn't read file:({0}): {1}".format(
e.errno, e.strerror))
logger.info("File {} I/O error({}): {}".format(
in_file, e.errno, e.strerror))
users = [
user['target_screen_name'] for user in json_data
if 'exists' in user and user['exists'] == 0
]
logger.debug("Got {} Non existant users".format(len(users)))
return users
def fetch_tweet_info(url, headers={'accept': 'application/json'}):
logger.debug("Fetching {} URL".format(url))
headers, response_json = make_api_request(url=url,
method='GET',
headers=headers)
global g_headers
g_headers = headers
if isinstance(response_json, dict) and 'errors' in response_json.keys():
errors = response_json['errors']
logger.error("Encountered error {} while accessing {} URL".format(
response_json, url))
for error in errors:
if 'code' in error.keys():
if error['code'] == 88:
raise TwitterRateLimitError(response_json)
elif error['code'] == 50:
raise TwitterUserNotFoundError(response_json)
raise Exception('Twitter API error: %s' % response_json)
return response_json