def _get_plugin_repos_from_dotfiles(repo_data, search_keyword): """Search for references to vim plugin repos from a dotfiles repository, and insert them into DB. Arguments: repo_data: API response from GitHub of a repository. search_keyword: The keyword used that found this repo. """ owner_repo = repo_data['full_name'] # Print w/o newline. print " scraping %s ..." % owner_repo, sys.stdout.flush() res, contents_data = get_api_page('repos/%s/contents' % owner_repo) if res.status_code == 404 or not isinstance(contents_data, list): print "contents not found" return repos_by_manager = _extract_bundle_repos_from_dir(contents_data) vundle_repos = repos_by_manager.vundle neobundle_repos = repos_by_manager.neobundle vimplug_repos = repos_by_manager.vimplug pathogen_repos = _extract_pathogen_repos(contents_data) owner, repo_name = owner_repo.split('/') db_repo = DotfilesGithubRepos.get_with_owner_repo(owner, repo_name) pushed_date = dateutil.parser.parse(repo_data['pushed_at']) def stringify_repo(owner_repo_tuple): return '/'.join(owner_repo_tuple) repo = dict( db_repo or {}, **{ 'owner': owner, 'pushed_at': util.to_timestamp(pushed_date), 'repo_name': repo_name, 'search_keyword': search_keyword, 'vundle_repos': map(stringify_repo, vundle_repos), 'neobundle_repos': map(stringify_repo, neobundle_repos), 'vimplug_repos': map(stringify_repo, vimplug_repos), 'pathogen_repos': map(stringify_repo, pathogen_repos), }) DotfilesGithubRepos.log_scrape(repo) DotfilesGithubRepos.upsert_with_owner_repo(repo) print 'found %s Vundles, %s NeoBundles, %s VimPlugs, %s Pathogens' % ( len(vundle_repos), len(neobundle_repos), len(vimplug_repos), len(pathogen_repos)) return { 'vundle_repos_count': len(vundle_repos), 'neobundle_repos_count': len(neobundle_repos), 'vimplug_repos_count': len(vimplug_repos), 'pathogen_repos_count': len(pathogen_repos), }
def _get_plugin_repos_from_dotfiles(repo_data, search_keyword): """Search for references to vim plugin repos from a dotfiles repository, and insert them into DB. Arguments: repo_data: API response from GitHub of a repository. search_keyword: The keyword used that found this repo. """ owner_repo = repo_data['full_name'] # Print w/o newline. print " scraping %s ..." % owner_repo, sys.stdout.flush() res, contents_data = get_api_page('repos/%s/contents' % owner_repo) if res.status_code == 404 or not isinstance(contents_data, list): print "contents not found" return repos_by_manager = _extract_bundle_repos_from_dir(contents_data) vundle_repos = repos_by_manager.vundle neobundle_repos = repos_by_manager.neobundle vimplug_repos = repos_by_manager.vimplug pathogen_repos = _extract_pathogen_repos(contents_data) owner, repo_name = owner_repo.split('/') db_repo = DotfilesGithubRepos.get_with_owner_repo(owner, repo_name) pushed_date = dateutil.parser.parse(repo_data['pushed_at']) def stringify_repo(owner_repo_tuple): return '/'.join(owner_repo_tuple) repo = dict(db_repo or {}, **{ 'owner': owner, 'pushed_at': util.to_timestamp(pushed_date), 'repo_name': repo_name, 'search_keyword': search_keyword, 'vundle_repos': map(stringify_repo, vundle_repos), 'neobundle_repos': map(stringify_repo, neobundle_repos), 'vimplug_repos': map(stringify_repo, vimplug_repos), 'pathogen_repos': map(stringify_repo, pathogen_repos), }) DotfilesGithubRepos.log_scrape(repo) DotfilesGithubRepos.upsert_with_owner_repo(repo) print 'found %s Vundles, %s NeoBundles, %s VimPlugs, %s Pathogens' % ( len(vundle_repos), len(neobundle_repos), len(vimplug_repos), len(pathogen_repos)) return { 'vundle_repos_count': len(vundle_repos), 'neobundle_repos_count': len(neobundle_repos), 'vimplug_repos_count': len(vimplug_repos), 'pathogen_repos_count': len(pathogen_repos), }