def add_contributors(repo, num_layers_to_crawl=0):
"""Add all contributors of repo to db."""
contributors = repo.get_contributors()
# num_contributors = repo.contributors_count
num_users = 0
count = 0
# msg = "Adding contributors: "#.format(num_contributors)
for contributor in contributors:
count += 1
if (num_users > config.MAX_CRAWL_COUNT_NEW
or count > config.MAX_CRAWL_COUNT_TOTAL):
break
# If contributor is in db, skip and continue.
# TODO: Consider checking last_crawled of contributor.repo/contributor.user.
this_contributor = Contributor.query.filter_by(
repo_id=repo.id, user_id=contributor.id).first()
if this_contributor:
#TODO: A queue might be more robust than a recursive process.
if num_layers_to_crawl:
crawl_from_user_to_repos(user, num_layers_to_crawl)
continue
num_users += add_user(contributor, num_layers_to_crawl)
this_contributor = Contributor(repo_id=repo.id, user_id=contributor.id)
db.session.add(this_contributor)
db.session.commit()
return num_users
def Contributor_add(av, name):
record = Contributor.gql('WHERE avkey = :1', av).get()
if record is None:
NewDist = Contributor(avkey = av, avname = name)
NewDist.put()
token = "contr_auth_%s" % av
memcache.set(token, True)
def example_data():
"""Create example data for the test database."""
Dislike.query.delete()
Stargazer.query.delete()
Watcher.query.delete()
Follower.query.delete()
Contributor.query.delete()
RepoLanguage.query.delete()
Language.query.delete()
Repo.query.delete()
Account.query.delete()
User.query.delete()
jane = User(user_id="1",
login="******",
name="Jane",
last_crawled=datetime.datetime.now(),
last_crawled_depth=2)
alex = User(user_id="2",
login="******",
name="Alex",
last_crawled=(datetime.datetime.now() -
datetime.timedelta(weeks=6)),
last_crawled_depth=2)
kelly = User(user_id="3", login="******", name="Kelly")
db.session.add_all([jane, alex, kelly])
db.session.commit()
jane_account = Account(user_id="1", access_token="abc123")
db.session.add(jane_account)
db.session.commit()
py_repo = Repo(repo_id="1",
name="python-repo",
description="A Python repository",
owner_id="1",
last_crawled=datetime.datetime.now(),
last_crawled_depth=2,
url="https://github.com/jhacks/python-repo",
stargazers_count=2)
js_repo = Repo(repo_id="2",
name="js-repo",
description="A Javascript repository",
owner_id="1",
last_crawled=(datetime.datetime.now() -
datetime.timedelta(weeks=6)),
last_crawled_depth=1,
url="https://github.com/jhacks/js-repo",
stargazers_count=1)
db.session.add_all([py_repo, js_repo])
db.session.commit()
astar = Stargazer(repo_id="1", user_id="2")
kstar = Stargazer(repo_id="1", user_id="3")
kstar_js = Stargazer(repo_id="2", user_id="3")
a_dislike_js = Dislike(repo_id="2", user_id="2")
# k_dislike_js = Dislike(repo_id="2", user_id="3")
db.session.add_all([astar, kstar, kstar_js, a_dislike_js])
db.session.commit()
kwatch = Watcher(repo_id="1", user_id="3")
a_j_follow = Follower(user_id="1", follower_id="2")
k_j_follow = Follower(user_id="1", follower_id="3")
j_a_follow = Follower(user_id="2", follower_id="1")
db.session.add_all([kwatch, a_j_follow, k_j_follow, j_a_follow])
db.session.commit()
jcon = Contributor(repo_id="1", user_id="1")
kcon = Contributor(repo_id="1", user_id="3")
db.session.add_all([jcon, kcon])
db.session.commit()
# python = Topic(topic_id="1", topic_name="python")
# api = Topic(topic_id="2", topic_name="api")
# db.session.add_all([python, api])
# db.session.commit()
# py_rep1 = RepoTopic(topic_id="1", repo_id="1")
# api_rep1 = RepoTopic(topic_id="2", repo_id="1")
# db.session.add_all([py_rep1, api_rep1])
# db.session.commit()
py_lang = Language(language_id="1", language_name="python")
c_lang = Language(language_id="2", language_name="c")
db.session.add_all([py_lang, c_lang])
db.session.commit()
py_lang_rep1 = RepoLanguage(language_id="1",
repo_id="1",
language_bytes=5000)
c_lang_rep1 = RepoLanguage(language_id="2",
repo_id="1",
language_bytes=100)
db.session.add_all([py_lang_rep1, c_lang_rep1])
db.session.commit()