class GithubSearch: def __init__(self, query, language, user, password): self.query = query self.language = language self.user = user self.password = password self.g = GitHub(self.user, self.password) def search(self, num_results = 2): return self.g.search_repositories(self.query + " language:" + self.language, number=num_results)
def clone_repositories():
gh = GitHub()
folderpath = "gitrepos"
repos = gh.search_repositories("cs188 language:python")
repo_count = 50
for r in repos:
if repo_count == 0: break
repo_json = r.to_json()
repo_path = os.path.join(folderpath, repo_json["full_name"])
if not mkdir(repo_path): continue
Repo.clone_from(repo_json["clone_url"], repo_path)
repo_count -= 1
return None
def get_gh_repos(user, repo_count):
"""
Get a user's repositories with a limit
:param user:
:param repo_count:
:return:
"""
repos = None
if user and repo_count:
try:
gh = GitHub()
repo_limit = int(repo_count) if repo_count else 5
repos = list(map(lambda r: r.repository, list(gh.search_repositories(
'user:'******'updated'))[:repo_limit]))
except GitHubError:
repos = None
return repos
def generate():
settings = json.load(open('settings.json'))
# Get all filenames
pages = _sorted_list_dir(os.path.join('content', 'pages'))
articles = _sorted_list_dir(os.path.join('content', 'articles'))
files = pages + articles
# BEGIN: Parse files
loader = jn.FileSystemLoader(os.path.join(os.getcwd(), 'templates'))
env = jn.Environment(loader=loader)
# Add GitHub repos
repos = None
if settings.get('github_user') and settings.get('github_repo_count'):
try:
gh = GitHub()
repo_limit_maybe = settings.get('github_repo_count')
repo_limit = int(repo_limit_maybe) if repo_limit_maybe else 5
repos = list(map(lambda r: r.repository, list(gh.search_repositories(
'user:'******'github_user'], sort='updated'))[:repo_limit]))
except GitHubError:
repos = None
print(
'GitHub search repositories error. Check "github_user" in settings.json')
print(' - Parsing articles and pages')
for f in files:
fp = f.open()
file_no_ext = f.name.split('.')[0]
rest, data = parse_metadata(fp)
html = markdown.markdown(''.join(rest))
target = data.get('layout') + 's'
target_dir = os.path.join('output', target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Create file in output direction
layout = data.get('layout')
template = env.get_template(layout + '.html')
data['content'] = html
data['url'] = target + '/' + file_no_ext + '.html'
data['site'] = settings
data['site']['recent_articles'] = get_recent_articles(articles)
if repos:
data['gh_repos'] = repos
page = template.render(data) # Date and other things
file_out = open(
os.path.join('output', target, file_no_ext + '.html'), 'w')
file_out.write(page)
file_out.close()
# Copy index
print(' - Parsing index page')
parse_index(os.path.join('content', 'index.md'), {
'env': env,
'settings': settings,
'articles': articles,
'repos': repos
})
# END: Parse files
# Copy static directory
print(' - Generating static directory')
static = os.path.join('output', 'static')
if os.path.exists(static):
shutil.rmtree(static)
_copy_directory('static', os.path.join('output', 'static'))
class GitHubClient:
def __init__(self, config):
self.config = config
self._client = GitHub(login=self.login, password=self.secret, token=self.token)
self.found_project = None
printer_config = self.config.copy()
printer_config['formatter'] = _Formatter
printer_config['writer'] = StdoutWriter
self.printer = Printer(printer_config)
def find_project(self):
if self.project_owner:
project = repository(repository=self.project_name, owner=self.project_owner)
if project:
self.found_project = project
return
self._error_nothing_found(True)
search_result = list(self._client.search_repositories(
'language:python {}'.format(self.project_name), number=10))
repositories_count = len(search_result)
if repositories_count == 1:
self.found_project = search_result[0].repository
self.printer.print('Found {}'.format(self.found_project))
return
if repositories_count == 0:
self._error_nothing_found()
repos = '\n'.join(['[{}]{}/{}'.format(i, r.repository.owner.login, r.repository.name)
for i, r in enumerate(search_result, start=1)])
message = 'Found {count} projects:\n{repos}\n'.format(count=repositories_count, repos=repos)
self.printer.print(message)
self.found_project = search_result[self._ask(repos)].repository
def _error_nothing_found(self, with_owner=False):
msg = 'Nothing was found! Check project name '
if with_owner:
msg += 'and repository owner '
msg += 'you are looking for!'
self.printer.print(msg)
exit(5)
def _ask(self, repos):
repositories_count = len(repos)
answer = None
while not answer:
tmp = input('Choose one of them or write "NO" to stop: ')
if tmp == 'NO':
exit(3)
try:
answer = int(tmp)
except ValueError:
self.printer.print('Write only "NO" or repo number!')
continue
if not (0 < answer <= repositories_count):
self.printer.print('Incorrect number!')
continue
return answer - 1
def download_project(self, archive_fd):
url = self.found_project.archive_urlt.expand(archive_format='zipball')
response = requests.get(url, stream=True)
shutil.copyfileobj(response.raw, archive_fd)
def unzip_project(self, archive_fd, project_folder):
with ZipFile(archive_fd) as zip:
zip.extractall(path=project_folder)
@property
def project_name(self):
return self.config['project-name']
@property
def project_owner(self):
return self.config.get('user')
@property
def login(self):
return self.config.get('login')
@property
def secret(self):
return self.config.get('secret')
@property
def token(self):
return self.config.get('token')
from github3 import GitHub
from git import Repo
from constants import REPOS_BASE_PATH
import os
gh = GitHub()
search_results = gh.search_repositories("dotfiles")
import itertools
for r in search_results:
repo_path = REPOS_BASE_PATH + r.repository.full_name
git_url = r.repository.clone_url
if not os.path.exists(repo_path):
os.makedirs(repo_path)
Repo.clone_from(git_url, repo_path)
print "Cloning into %s" % repo_path
else:
print "Already have %s" % repo_path
