def retrieve_repos_by_keyword(keywords, language, username="", password=""):
gh = None
if username and password:
gh = login(username, password=password)
else:
gh = GitHub()
try:
for keyword in keywords:
for page in range(1, 11):
repos = gh.search_repos(keyword, language=language, start_page=page)
for repo in repos:
r = repo.to_json()
if r["language"] == language:
result = r
try:
user = gh.user(r["owner"]).to_json()
result["owner_email"] = user.get("email", "")
result["owner_blog"] = user.get("blog", "")
result["owner_location"] = user.get("location", "")
result["owner_avatar"] = user.get("avatar_url", "")
except Exception as e:
pass
yield result
except Exception as e:
print e
pass
class GitHubHandler(BaseHandler):
def __init__(self):
if settings.GITHUB_TOKEN:
self.github = login(token=settings.GITHUB_TOKEN)
else:
self.github = GitHub()
def get_account(self, name):
return self.github.user(name)
class GitHubDB(object):
def __init__(self, ghtoken):
# Get handle to Github API
if ghtoken is not None and ghtoken != '':
self.gh = login(token=ghtoken)
else:
log.warning('Using unauthenticated access to Github API. This will result in severe rate limiting.')
self.gh = GitHub()
def waitForRateLimit(self, resourceType):
"""resourceType can be 'search' or 'core'."""
try:
rateLimitInfo = self.gh.rate_limit()['resources']
while rateLimitInfo[resourceType]['remaining'] < (1 if resourceType == 'search' else 12):
waitTime = max(1, rateLimitInfo[resourceType]['reset'] - time.time())
log.warning('Waiting %s seconds for Github rate limit...', waitTime)
time.sleep(waitTime)
rateLimitInfo = self.gh.rate_limit()['resources']
except ConnectionError as e:
log.error("Connection error while querying GitHub rate limit. Retrying...")
self.waitForRateLimit(resourceType)
def refreshGithubUser(self, ghUserObject):
self.waitForRateLimit('core')
return ghUserObject.refresh(True)
def getGithubUserForLogin(self, login, session):
"""Uses the Github API to find the user for the given username. Returns NullObject if the user was not found for any reason."""
# Try to use cached result to avoid hitting rate limit
cachedUser = session.query(GitHubUserCache).filter(GitHubUserCache.login == login).first()
if cachedUser is not None:
return cachedUser if not cachedUser.fake else NullObject()
log.debug('Querying GutHub API for login %s', login)
try:
self.waitForRateLimit('core')
potentialUser = self.gh.user(login)
if potentialUser is None:
# store login as fake
session.add(GitHubUserCache(login=login, fake=True))
return NullObject()
actualUser = self.refreshGithubUser(potentialUser)
if isinstance(potentialUser, NullObject):
# store login as fake
session.add(GitHubUserCache(login=login, fake=True))
else:
# cache user
session.add(GitHubUserCache(login=login, name=actualUser.name, email=actualUser.email, company=actualUser.company, location=actualUser.location))
return actualUser
except ConnectionError:
log.error("github query failed when attempting to verify username %s", login)
return NullObject()
def searchGithubUsers(self, query):
self.waitForRateLimit('search')
return self.gh.search_users(query)
def get_cached_user(gh: GitHub, username: str) -> users.User:
"""
Get a GitHub user by username. Results are cached to stay under API limits.
"""
key = f"gh_user_{username}"
user_dict = cache.get(key)
if user_dict is not None:
return users.User.from_dict(user_dict, gh.session)
user = gh.user(username)
cache.set(key, user.as_dict(), timeout=60 * 60 * 24) # 1 day
return user
def determine_email_address(
github_user_name: str,
github_api: GitHub,
) -> typing.Optional[str]:
not_none(github_user_name)
try:
user = github_api.user(github_user_name)
except NotFoundError:
logger.warning(f'failed to lookup {github_user_name=} {github_api._github_url=}')
return None
return user.email
def callAnonymous():
# anonymous usage
gh = GitHub()
usrQG = gh.user('QueensGambit')
repo = gh.repository('QueensGambit', 'Barcode-App')
print('usrQG.msg:')
print(usrQG)
print('repos:')
print(repo)
lstEmojis = gh.emojis()
print('lstEmojis:')
print(lstEmojis)
def create(self, validated_data):
gh = GitHub()
github_name = validated_data['github']
user_data = validated_data.pop('user')
user_data['username'] = user_data.get('email')
if github_name:
gh_id = gh.user(github_name).id
validated_data['github_id'] = gh_id
user = User.objects.create(**user_data)
member = Member.objects.create(user=user, **validated_data)
return member
def create(self, request, course_pk=None):
gh = GitHub()
course = Course.objects.get(pk=course_pk)
members = request.FILES["members"]
if members:
csvf = StringIO(members.read().decode())
reader = csv.DictReader(csvf, delimiter=",")
for row in reader:
password = User.objects.make_random_password()
user = User.objects.create_user(
first_name=row[CSV_FORMAT["first_name"]],
last_name=row[CSV_FORMAT["last_name"]],
email=row[CSV_FORMAT["email"]],
username=row[CSV_FORMAT["email"]],
)
user.set_password(password)
user.save()
github_user = gh.user(row[CSV_FORMAT["github"]])
member = Member.objects.create(
user=user,
github=row[CSV_FORMAT["github"]],
github_id=github_user.id,
student_class=row[CSV_FORMAT["student_class"]],
student_grade=row[CSV_FORMAT["student_grade"]],
student_number=row[CSV_FORMAT["student_number"]],
)
membership = Membership.objects.create(member=member, course=course, role="S")
send_enroll_email(course, user, password)
return HttpResponse("", status=status.HTTP_204_NO_CONTENT)
class Command(object):
__metaclass__ = ABCMeta
name = None
usage = None
repository = ()
user = ''
subcommands = {}
SUCCESS = 0
FAILURE = 1
COMMAND_UNKNOWN = 127
def __init__(self):
super(Command, self).__init__()
assert self.name
commands[self.name] = self
self.gh = GitHub()
self.gh.set_user_agent('github-cli/{0} (http://git.io/MEmEmw)'.format(
__version__
))
self.parser = CustomOptionParser(usage=self.usage)
@abstractmethod
def run(self, options, args):
return self.FAILURE
def get_repo(self, options):
self.repo = None
if self.repository:
self.repo = self.gh.repository(*self.repository)
if not (self.repo or options.loc_aware):
self.parser.error('A repository is required.')
def get_user(self):
if not self.user:
self.login()
self.user = self.gh.user()
def login(self):
# Get the full path to the configuration file
config = github_config()
parser = ConfigParser()
# Check to make sure the file exists and we are allowed to read it
if os.path.isfile(config) and os.access(config, os.R_OK | os.W_OK):
parser.readfp(open(config))
self.gh.login(token=parser.get('github', 'token'))
else:
# Either the file didn't exist or we didn't have the correct
# permissions
user = ''
while not user:
# We will not stop until we are given a username
user = input('Username: '******''
while not pw:
# Nor will we stop until we're given a password
pw = getpass('Password: '******'user', 'repo', 'gist'], 'github-cli',
'http://git.io/MEmEmw'
)
parser.add_section('github')
parser.set('github', 'token', auth.token)
self.gh.login(token=auth.token)
# Create the file if it doesn't exist. Otherwise completely blank
# out what was there before. Kind of dangerous and destructive but
# somewhat necessary
parser.write(open(config, 'w+'))
def help(self):
self.parser.print_help()
if self.subcommands:
print('\nSubcommands:')
for command in sorted(self.subcommands.keys()):
print(' {0}:\n\t{1}'.format(
command, self.subcommands[command]
))
sys.exit(0)
import json
import argparse
from github3 import login
from github3 import GitHub
import base64
app = Flask(__name__)
f=sys.argv[1]
g=GitHub()
urlcon=f.split('/')
l=len(urlcon)
print(urlcon[l-1])
print(urlcon[l-2])
amiforgit=g.user('amiforgit')
@app.route("/v1/<name>")
def hello1(name):
if name.endswith('.yml'):
retval=base64.b64decode(g.repository(urlcon[l-2],urlcon[l-1]).contents(name).content)
return retval
if name.endswith('.yaml'):
name=name[:-5]+".yml"
retval=base64.b64decode(g.repository(urlcon[l-2],urlcon[l-1]).contents(name).content)
return retval
if name.endswith('.json'):
name=name[:-5]+".yml"
print(name)
retval=base64.b64decode(g.repository(urlcon[l-2],urlcon[l-1]).contents(name).content)
retvalf=(json.dumps(yaml.load(retval), sort_keys=False, indent=2))
def starred(username, token, sort, repository, message, format):
"""GitHub starred
creating your own Awesome List used GitHub stars!
example:
starred --username ntk148v --sort > README.md
"""
if repository:
if not token:
click.secho('Error: create repository need set --token', fg='red')
return
file = BytesIO()
sys.stdout = file
else:
file = None
gh = GitHub(token=token)
user = gh.user(username)
stars = gh.starred_by(username)
click.echo(desc)
repo_dict = {}
# Just a dict to store index
repo_num_dict = {}
for s in stars:
language = s.language or 'Others'
description = html_escape(s.description).replace(
'\n', '') if s.description else ''
if language not in repo_dict:
repo_dict[language] = []
repo_num_dict[language] = 1
repo_dict[language].append(
[repo_num_dict[language], s.name, s.html_url,
description.strip()])
repo_num_dict[language] += 1
if sort:
repo_dict = OrderedDict(sorted(repo_dict.items(), key=lambda l: l[0]))
for language in repo_dict.keys():
data = u' - [{}](#{})'.format(language,
'-'.join(language.lower().split()))
click.echo(data)
click.echo('')
for language in repo_dict:
click.echo('## {} \n'.format(language.replace('#', '# #')))
if format == 'table':
writer = MarkdownTableWriter(
headers=['Index', 'Name', 'Repository URL', 'Description'],
value_matrix=repo_dict[language],
margin=1)
click.echo(writer.dumps())
else:
for repo in repo_dict[language]:
data = u'{}. [{}]({}) - {}'.format(*repo)
click.echo(data)
click.echo('')
click.echo(license_.format(username=username))
if file:
try:
rep = gh.repository(username, repository)
readme = rep.readme()
readme.update(message,
file.getvalue(),
author={
'name': user.name,
'email': user.email
})
except NotFoundError:
rep = gh.create_repository(repository,
'A curated list of my GitHub stars!')
rep.create_file('README.md',
'starred initial commit',
file.getvalue(),
author={
'name': user.name,
'email': user.email
})
click.launch(rep.html_url)
from config import github as config
from db import DBFeedItem, DBConnection
from github3 import GitHub
from requests.exceptions import ConnectionError
import logging
import datetime
gh = GitHub(config['username'], config['password'])
events = gh.user(config['username']).iter_received_events(public = True)
TYPE = 'github'
_logger = logging.getLogger(TYPE)
conn = DBConnection()
accepted_types = [m for m in config['accepted_types'] if config['accepted_types'][m] == True]
type_header_map = {
'CommitCommentEvent': '{event.actor.login} commented on commit {event.repo[0]}/{event.repo[1]}@{rev_10}',
'CreateEvent': '{event.actor.login} created {ref_type} {var_created}',
# 'DeleteEvent': 'A {ref_type} was deleted.', # TODO: format like github does
'ForkEvent': '{event.actor.login} forked {event.repo[0]}/{event.repo[1]} to {forkee.full_name}',
'GollumEvent': '{event.actor.login} edited the {event.repo[0]}/{event.repo[1]} wiki',
'IssueCommentEvent': '{event.actor.login} commented on issue {event.repo[0]}/{event.repo[1]}#{issue.number}',
'IssuesEvent': '{event.actor.login} {action} issue {event.repo[0]}/{event.repo[1]}#{issue.number}',
'MemberEvent': '{event.actor.login} {action} {added_user.login} to {event.repo[0]}/{event.repo[1]}',
'PublicEvent': '{event.actor.login} made {event.repo[0]}/{event.repo[1]} public',
'PullRequestEvent': '{event.actor.login} {action} pull request {event.repo[0]}/{event.repo[1]}#{number}',
'PullRequestReviewCommentEvent': '{event.actor.login} commented on pull request {event.repo[0]}/{event.repo[1]}#{number}',
'PushEvent': '{event.actor.login} pushed to {ref} at {event.repo[0]}/{event.repo[1]}',
# 'ReleaseEvent': 'A new comment was given on a commit.', # TODO: format like github does
'WatchEvent': '{event.actor.login} starred {event.repo[0]}/{event.repo[1]}',
def getGitHubProfiles(locations, languages, num):
logger.info("Locations: {0}".format(locations))
logger.info("Languages: {0}".format(languages))
num = int(num) if num else DEFAULT_DESIRED_CANDIDATES_PER_EMAIL_DIGEST
logger.info("Number of Profiles requested: {0}".format(num))
logger.info("Building query string")
queryString = ''
for location in locations:
queryString = queryString + 'location:\"' + location + '\" '
for language in languages:
queryString = queryString + 'language:\"' + language + '\" '
queryString = queryString + 'type:User'
logger.info("Query String = {}".format(queryString))
logger.info("Connecting to Github")
gh = GitHub(token=os.environ['TOKEN'])
logger.info("Getting a list of matching users using GitHub API")
matchingUsers = []
for userSearchResult in gh.search_users(queryString):
matchingUsers.append(userSearchResult.user)
logger.info("Number of matching profiles: {}".format(len(matchingUsers)))
userActivityDict = {}
logger.info(
"Using githubcontributions api to get the number of contributions for each user"
)
# TODO: Remove the top 25 when ready
for u in matchingUsers[:25]:
cmd = 'curl -s https://githubcontributions.io/api/user/' + u.login
output = subprocess.check_output(cmd, shell=True)
userActivityDict[u.login] = json.loads(output)['eventCount']
logger.info("Sorting the profiles based on # of contributions")
topUsers = sorted(userActivityDict.items(),
key=lambda x: x[1],
reverse=True)
logger.info(
"Emailing top {} profiles not already in the cache (not already sent before)"
.format(num))
r = redis.StrictRedis(host='localhost', port=6379, db=0)
format.initialize(num)
# TODO Run the following when done debugging, to clear the cache
# redis-cli flushall
count = 0
for u in topUsers:
if count < num:
usr = gh.user(u[0])
contributions = u[1]
if not r.exists(usr.login) and (usr.company == None
or 'HookLogic' not in usr.company
or 'Hooklogic' not in usr.company):
# Query StackExchange for User id
cmd = 'curl -s http://data.stackexchange.com/stackoverflow/csv/670133?Name=' + usr.login
output = subprocess.check_output(cmd, shell=True)
user_id = ''
user_id = output.split('\n')[1].replace('\"', '')
stackoverflow_url = "http://stackoverflow.com/users/" + user_id + "/" + usr.login
format.format_html(usr, contributions,
stackoverflow_url if user_id else '')
r.set(usr.login, True)
count = count + 1
format.save_file()
send_email.send()
class Github:
def __init__(self, _ask_credentials=None, _ask_2fa=None):
self.last_error = None
self._ask_credentials = _ask_credentials
self._ask_2fa = _ask_2fa
self.gh = GitHub(token=self._get_authorization_token())
self.user = self.gh.user()
def _get_authorization_token(self):
if not config.has('github', 'token') or \
not config.get('github', 'token'):
if not self._ask_credentials:
raise RuntimeError('Github Token is not set in the '
'configuration and no function was set to '
'ask for credentials')
token = self._gen_authorization_token()
config.set('github', 'token', token)
config.save()
return config.get('github', 'token')
def _gen_authorization_token(self, counter=0, creds=None):
"""
This function creates the authorization token for AerisCloud.
If an existing token exists for this computer, it adds a #N counter
next to the name.
"""
if creds:
user, pwd = creds['user'], creds['pwd']
else:
(user, pwd) = self._ask_credentials()
note = 'AerisCloud on %s' % (node())
if counter > 0:
note += ' #%d' % counter
try:
auth = authorize(user, pwd, ['repo', 'read:org'], note=note,
two_factor_callback=self._ask_2fa)
return auth.token
except GitHubError as e:
if not e.errors or e.errors[0]['code'] != 'already_exists':
raise
# token exists, increment counter
counter += 1
return self._gen_authorization_token(counter=counter,
creds={'user': user,
'pwd': pwd})
def get_organizations(self):
return [org for org in self.gh.iter_orgs()]
def get_repo(self, name, fork=False):
"""
Find a repository in the available ogranizations, if fork is set to
True it will try forking it to the user's account
"""
# check on the user
repo = self.gh.repository(self.user.login, name)
if repo:
return repo, None
if not config.has('github', 'organizations'):
return False, None
# then on configured organization
organizations = config.get('github', 'organizations').split(',')
for org in organizations:
repo = self.gh.repository(org, name)
if repo:
break
if not repo:
return False, None
if not fork:
return repo, None
return repo.create_fork(), repo
