def collect_data(username):
client = get_twitter_client()
query = {"username": username}
queryRes = posts.find_one(query)
if queryRes is None:
for page in Cursor(client.user_timeline,
screen_name=username,
count=200).pages(16):
for status in page:
post_data = {'username': username, 'content': status._json}
posts.insert_one(post_data)
def paginate(items, n):
"""
Generate n-sized chunks from Items
"""
for i in range(0, len(items), n):
yield items[i:i + n]
if __name__ == '__main__':
if len(sys.argv) != 2:
usage()
sys.exit(1)
screen_name = sys.argv[1]
client = get_twitter_client()
dirname = "Twitter_Profiles/{}".format(screen_name)
max_pages = math.ceil(MAX_FRIENDS / 5000)
try:
os.makedirs(dirname, mode=0o755, exist_ok=True)
except OSError:
print("Directory {} already exists".format(dirname))
except Exception as e:
print("Error while creating directory {}".format(dirname))
print(e)
sys.exit(1)
#get followers for a given user
fname = "Twitter_Profiles/{}/followers.jsonl".format(screen_name)
with open(fname, 'w') as f: