def _get_plugin_repos_from_dotfiles(repo_data, search_keyword):
"""Search for references to vim plugin repos from a dotfiles repository,
and insert them into DB.
Arguments:
repo_data: API response from GitHub of a repository.
search_keyword: The keyword used that found this repo.
"""
owner_repo = repo_data['full_name']
# Print w/o newline.
print " scraping %s ..." % owner_repo,
sys.stdout.flush()
res, contents_data = get_api_page('repos/%s/contents' % owner_repo)
if res.status_code == 404 or not isinstance(contents_data, list):
print "contents not found"
return
repos_by_manager = _extract_bundle_repos_from_dir(contents_data)
vundle_repos = repos_by_manager.vundle
neobundle_repos = repos_by_manager.neobundle
vimplug_repos = repos_by_manager.vimplug
pathogen_repos = _extract_pathogen_repos(contents_data)
owner, repo_name = owner_repo.split('/')
db_repo = DotfilesGithubRepos.get_with_owner_repo(owner, repo_name)
pushed_date = dateutil.parser.parse(repo_data['pushed_at'])
def stringify_repo(owner_repo_tuple):
return '/'.join(owner_repo_tuple)
repo = dict(
db_repo or {}, **{
'owner': owner,
'pushed_at': util.to_timestamp(pushed_date),
'repo_name': repo_name,
'search_keyword': search_keyword,
'vundle_repos': map(stringify_repo, vundle_repos),
'neobundle_repos': map(stringify_repo, neobundle_repos),
'vimplug_repos': map(stringify_repo, vimplug_repos),
'pathogen_repos': map(stringify_repo, pathogen_repos),
})
DotfilesGithubRepos.log_scrape(repo)
DotfilesGithubRepos.upsert_with_owner_repo(repo)
print 'found %s Vundles, %s NeoBundles, %s VimPlugs, %s Pathogens' % (
len(vundle_repos), len(neobundle_repos), len(vimplug_repos),
len(pathogen_repos))
return {
'vundle_repos_count': len(vundle_repos),
'neobundle_repos_count': len(neobundle_repos),
'vimplug_repos_count': len(vimplug_repos),
'pathogen_repos_count': len(pathogen_repos),
}
def _get_plugin_repos_from_dotfiles(repo_data, search_keyword):
"""Search for references to vim plugin repos from a dotfiles repository,
and insert them into DB.
Arguments:
repo_data: API response from GitHub of a repository.
search_keyword: The keyword used that found this repo.
"""
owner_repo = repo_data['full_name']
# Print w/o newline.
print " scraping %s ..." % owner_repo,
sys.stdout.flush()
res, contents_data = get_api_page('repos/%s/contents' % owner_repo)
if res.status_code == 404 or not isinstance(contents_data, list):
print "contents not found"
return
repos_by_manager = _extract_bundle_repos_from_dir(contents_data)
vundle_repos = repos_by_manager.vundle
neobundle_repos = repos_by_manager.neobundle
vimplug_repos = repos_by_manager.vimplug
pathogen_repos = _extract_pathogen_repos(contents_data)
owner, repo_name = owner_repo.split('/')
db_repo = DotfilesGithubRepos.get_with_owner_repo(owner, repo_name)
pushed_date = dateutil.parser.parse(repo_data['pushed_at'])
def stringify_repo(owner_repo_tuple):
return '/'.join(owner_repo_tuple)
repo = dict(db_repo or {}, **{
'owner': owner,
'pushed_at': util.to_timestamp(pushed_date),
'repo_name': repo_name,
'search_keyword': search_keyword,
'vundle_repos': map(stringify_repo, vundle_repos),
'neobundle_repos': map(stringify_repo, neobundle_repos),
'vimplug_repos': map(stringify_repo, vimplug_repos),
'pathogen_repos': map(stringify_repo, pathogen_repos),
})
DotfilesGithubRepos.log_scrape(repo)
DotfilesGithubRepos.upsert_with_owner_repo(repo)
print 'found %s Vundles, %s NeoBundles, %s VimPlugs, %s Pathogens' % (
len(vundle_repos), len(neobundle_repos),
len(vimplug_repos), len(pathogen_repos))
return {
'vundle_repos_count': len(vundle_repos),
'neobundle_repos_count': len(neobundle_repos),
'vimplug_repos_count': len(vimplug_repos),
'pathogen_repos_count': len(pathogen_repos),
}
