def store_users(github_cli: GithubClient, users):
"""
This callback stores user data in our gcp datastore database
@params: users array
"""
print("Storing page users data...")
for u in users:
if not u:
print("Failed to save user data.")
continue
print("Fetch detailed user {}:{} data...".format(u["id"], u["login"]))
user = github_cli.get_user(u["login"])
if github_cli.request_failed(user):
print("Failed to save user data: {} .".format(user))
return
print("Storing user {}:{} data...".format(u["id"], u["login"]))
convert_time_fields_to_date_time(user)
store_user(user)
print("Stored user {}:{} data.".format(u["id"], u["login"]))
print("-----------\n")
print("Page user data stored.")
def scrape_users(prs):
if prs.user_name != "":
print("[+] Getting dev information about: {}".format(prs.user_name))
user = get_user(prs.user_name)
if request_failed(user):
print("Failed to save user data: {} .".format(user))
else:
convert_time_fields_to_date_time(user)
store_user(user)
else:
print("[+] Getting devs from cameroun/cameroon...")
print("[+] pagination_limit: {}".format(prs.pagination_limit))
users = get_users(prs.pagination_limit, store_users, on_pageloaded_error)
print(users)
import argparse
import json
from app.utils.storage import store_user, store_project
if __name__ == "__main__":
# Usage :
# - setup-datastore:
# python3 -m app.cli
with open('./fixtures/users.json') as file:
data = json.load(file)
for d in data:
print("storing ", d["login"], "data...")
store_user(d)
print(d["login"], "data stored!\n")
with open('./fixtures/projects.json') as file:
data = json.load(file)
for d in data:
print("storing ", d["name"], " project data...")
store_project(d)
print(d["name"], "data project stored!\n")