def main(args):
username = None
password = None
fetchGists = False
# Parse options
opts, _ = getopt.getopt(args, 'gau:', ['gists', 'auth', 'user='******'-a', '--auth'):
username = raw_input('Username? ')
password = getpass.getpass('Password? ')
elif opt in ('-u', '--user'):
if ':' in arg:
username, password = arg.split(':')
else:
username = arg
elif opt in ('-g', '--gists'):
fetchGists = True
# Exit if no username set
if not username:
print 'Please provide a username with -u or ask for a username prompt with -a.'
sys.exit(0)
# Authenticate if password set
if not password:
API = github.GitHub()
if fetchGists:
repos = API.users(username).gists()
else:
repos = API.users(username).repos()
# Anonymous if no password set
else:
API = github.GitHub(username=username, password=password)
if fetchGists:
repos = API.gists()
else:
repos = API.user().repos()
# Iterate repos and clone
repos = repos.get()
for repo in repos:
if fetchGists:
url = repo.git_pull_url
path = repo.id
else:
url = repo.ssh_url
path = repo.name
# Don't clone if it already exists in this directory
if not os.path.exists(path):
_shell('git clone {}'.format(url))
else:
print '{} exists, aborting clone'.format(path)
def get_top_contributor_in_past_week(repo_link, username=None, password=None):
"""
Return top contributor of within the last week
Args:
repo_link (str) : the repository link in the format owner/repo_name
username (str): github username
password (str): github password
Returns:
dict: { 'username': <username>,
'name': <name>,
'commits': <commits>
'additions': <additions>
'deletions': <deletions>
}
"""
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
top = _get_contributor_stats(repo_link, gh)[-1]
return {
'username': top['author']['login'],
'name': get_name_from_username(top['author']['login']),
'commits': top['weeks'][0]['c'],
'additions': top['weeks'][0]['a'],
'deletions': top['weeks'][0]['d']
}
def get_latest_commit_summary(repo_link, username=None, password=None):
"""
Extracts latest commit info for a given repository.
Args:
repo_link (str) : the repository link in the format owner/repo_name
username (str): github username
password (str): github password
Returns:
dict: ['user'] --> {'name', 'email', 'username'}, ['message'], ['timestamp']
"""
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
latest_commit = _get_commits_from_api(repo_link, gh)[0]
latest_commit_dict = {}
latest_commit_dict['user'] = {}
latest_commit_dict['user']['name'] = latest_commit['commit']['author'][
'name']
latest_commit_dict['user']['email'] = latest_commit['commit']['author'][
'email']
latest_commit_dict['user']['username'] = latest_commit['committer'][
'login']
latest_commit_dict['message'] = latest_commit['commit']['message']
latest_commit_dict['timestamp'] = latest_commit['commit']['committer'][
'date']
return latest_commit_dict
def get_top_n_contributors(repo_link, n, username=None, password=None):
"""
Extracts top contributors for a given repository.
Args:
repo_link (str) : the repository link in the format owner/repo_name
n (int) : top n contributors; must be greater than 0
Returns:
list: top contributors in descending order of contributions
the n-th element of the list is the top (n+1)th contributor
each element is a dict containing ['username'] and ['contributions']
"""
assert n > 0
persons = 0
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
contributors = _get_contributors_from_api(repo_link, gh)
contributions = []
for contributor in contributors:
this_contributor = {}
this_contributor['username'] = contributor['login']
this_contributor['contributions'] = contributor['contributions']
contributions.append(this_contributor)
persons += 1
# only show top n contributors
if persons >= n:
break
return contributions
def get_stats_by_author(repo_link, author_name, username=None, password=None):
"""
Return total number of commits, lines added and lines delted by an author
Args:
repo_link : owner/repo format
author_name : limit history to one author
username (str) : github username
password (str) : github password
Return:
three numbers: commits, additions, deletions
"""
owner, repo = process_repo_link(repo_link)
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
repo_data = gh.repos(owner)(repo).stats.contributors.get()
for all_data in repo_data:
if all_data['author']['login'] == author_name:
total_commits = all_data['total']
weekly_data = all_data['weeks']
adds = 0
dels = 0
for week in weekly_data:
adds += week['a']
dels += week['d']
return total_commits, adds, dels
def main():
"""Main function. Read the configuration file, instantiate the GitHub object and run through checks"""
log = logger.Logger()
# Read the organization and Oauth token from the configuration file
log.log("Reading configuration")
configs = readConfig("config.ini")
# Instantiate the github object with the organization and Oauth token from the configuratio file
log.log("Instantiating github object")
gh = github.GitHub(configs['org'], configs['token'])
# Test 1: Check users without 2FA
log.log("Running first check: Check users without 2FA enabled")
check.check2FA(gh, log)
# Test 2: Check for invalid commit signatures
log.log("Running second check: Check Invalid Commit Signatures")
check.checkCommitSignature(gh, log)
# Test 3: Check Installed Webhooks
log.log("Running third check: Check installed webhooks")
check.checkInstalledWebhooks(gh, log)
log.log("Running fourth check: Check team member changes")
check.checkTeamMemberChanges(gh, log)
def getgithub(message, CMD, REPOS):
# -*- coding: utf-8 -*-
import github
import sys
import os
# os.environ でAPI TOKEN 取得
GITHUB_ACCESS_TOKEN = os.environ.get('GITHUB_ACCESS_TOKEN', '')
gh = github.GitHub(access_token=GITHUB_ACCESS_TOKEN)
# API TOLKEM入力後にusername情報取得
getuser = gh.user.get()
USER = getuser['name']
if CMD == "issue":
issues = gh.repos(USER)(REPOS).issues().get()
print("issues: " + issues[0]['state'])
print("issues: " + issues[0]['url'])
text = issues[0]['url']
elif CMD == "pull":
pulls = gh.repos(USER)(REPOS).pulls().get()
print("pulls: " + pulls[0]['review_comments_url'])
for key, value in getuser.items():
print(key, value)
text = pulls[0]['review_comments_url']
else:
text = ("usage:get [issue or pull] [REPOS]")
message.send(text)
def is_repo_link_valid(repo_link, username=None, password=None):
"""
Repository link validator.
Args:
repo_link (str): the repository link in the format owner/repo_name
Returns:
bool: True if repository exists (valid owner and repo_name) AND repository is public, False otherwise
"""
owner = repo = ''
try:
owner, repo = process_repo_link(repo_link)
except ValueError:
return False
try:
gh = github.GitHub(
username=username,
password=password) if username and password else GITHUB
gh.repos(owner)(repo).get()
return True
except github.ApiNotFoundError:
return False
def collect(user, repo, token, org):
if org is None:
org = user
db = __load_db(repo=repo)
gh = github.GitHub(access_token=token)
try:
gh.repos(org, repo).get()
except Exception:
sys.exit('No Data')
if user is not None and org != user:
try:
gh.repos(org, repo).collaborators(user).get()
except Exception:
sys.exit('No Data')
views_data = gh.repos(org, repo).traffic.views.get()
found_new_data = False
for view_per_day in views_data['views']:
timestamp = view_per_day['timestamp']
data = {'uniques': view_per_day['uniques'], 'count': view_per_day['count']}
if db.get(timestamp) is None or db.get(timestamp) is False:
db.set(timestamp, json.dumps(data))
print(timestamp, data)
found_new_data = True
else:
db_data = json.loads(db.get(timestamp))
if db_data['uniques'] < data['uniques']:
db.set(timestamp, json.dumps(data))
print(timestamp, data)
found_new_data = True
if not found_new_data:
print('No new traffic data was found for ' + org + '/' + repo)
db.dump()
def reportIssue(self, username, password, title, body):
if username == '' or password == '' or title == '' or body == '':
raise ValueError('incorrect parameters,try again.')
gh = github.GitHub(username, password)
gh.repos('racaljk')('pyann').issues.post(title=title,
body=body,
label='QuickIssue')
print 'You have succeed to report a Issue to hosts project via getHost script.'
def get_repo_contributors(repo_link, username=None, password=None):
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
contributors = _get_contributors_from_api(repo_link, gh)
contributor_list = []
for contributor in contributors:
contributor_list.append(contributor['login'])
return contributor_list
def test___init__(self):
init = github.GitHub("me", "arcade", "git.com")
self.assertEqual(init.user, "me")
self.assertEqual(init.token, "arcade")
self.assertEqual(init.api.auth, ("me", "arcade"))
self.assertEqual(init.url, "git.com")
def _ghp(self, expUrl, u, t, **kv):
def opener(url, data):
h = {'login': u, 'token': t}
h.update(kv)
self.assertEquals(github.BaseEndpoint.BASE_URL + expUrl, url)
self.assertEquals(sorted(data.split('&')),
sorted(urllib.urlencode(h).split('&')))
return github.GitHub(u, t, fetcher=opener)
def github_client():
"""
A fixture to initialize a GitHub API client, using the personal access token
obtained from the GITHUB_TOKEN environment variable.
"""
try:
token = os.environ["GITHUB_TOKEN"]
except KeyError:
raise Exception("To use any of the GitHub fixtures, you must set the GITHUB_TOKEN environment variable "
"to contain a GitHub personal access token.")
return github.GitHub(token)
def main(token, repo, db):
logging.basicConfig(level=logging.INFO)
hub = github.GitHub(access_token=token)
with sqlite3.connect(db) as rdb:
rdb.row_factory = row_factory
for loader in LOADERS:
rdb.cursor().execute(loader.schema)
for r in repo:
loader(hub, rdb, r)
def update_repository(request, pk):
r = get_object_or_404(request.user.access, pk=pk)
token = request.user.social_auth.filter(
provider='github').first().extra_data['access_token']
gh = github.GitHub(access_token=token)
# stat thread for downloading new info
t = threading.Thread(target=download_repo_info,
args=(gh, r.owner, r.name, r, True))
t.start()
return redirect("repo_list")
def authenticate(self):
"""
(class) -> boolean
Passing throught authentication and verifying if the right credentials were given.
"""
if self.personal_access_token is not None:
self.gh = github.GitHub(access_token = self.personal_access_token)
else:
self.gh = github.GitHub(
self.consumer_key,
self.consumer_secret,
self.access_token,
self.access_secret)
try:
some_get = self.gh.users(self.user_name).get()
print json.dumps(some_get, sort_keys=True, indent=4, separators=(',', ': '))
return True
except Exception, ex:
print '[e] exception {}'.format(str(ex))
return False
def collect(user, repo, org):
"""
Function for Collection of Data.
Counts the total number of views (Unique and total) on a Repo.
Parameters:
user (str): The Github Username
repo (str): The Name of Repo
org (str): The Name of Organization [Optional]
"""
token = decouple.config("API_KEY")
if org is None:
org = user
db = __load_db(repo=repo)
gh = github.GitHub(access_token=token)
try:
gh.repos(org, repo).get()
except Exception:
sys.exit("No Data")
if user is not None and org != user:
try:
gh.repos(org, repo).collaborators(user).get()
except Exception:
sys.exit("No Data")
views_data = gh.repos(org, repo).traffic.views.get()
found_new_data = False
for view_per_day in views_data["views"]:
timestamp = view_per_day["timestamp"]
data = {
"uniques": view_per_day["uniques"],
"count": view_per_day["count"]
}
if db.get(timestamp) is None or db.get(timestamp) is False:
db.set(timestamp, json.dumps(data))
print(timestamp, data)
found_new_data = True
else:
db_data = json.loads(db.get(timestamp))
if db_data["uniques"] < data["uniques"]:
db.set(timestamp, json.dumps(data))
print(timestamp, data)
found_new_data = True
if not found_new_data:
print("No new traffic data was found for " + org + "/" + repo)
db.dump()
def _collect(token, org, repo):
gh = github.GitHub(access_token=token)
repositries_data = gh.orgs(org).repos.get(per_page="1000")
print("total no. of public repository: ", len(repositries_data))
for rec in repositries_data:
repo_nm = rec['name']
forks = rec['forks']
stargazers_count = rec['stargazers_count']
watchers_count = rec['watchers_count']
print("processing repo: ", repo_nm)
commit_since = last_recorded_date(repo, repo_nm)
if commit_since == None:
commit_since = (dt.utcnow() - td(14)).date()
page_no = 1
commit_info = {}
commit_count_dict = {}
while len(commit_info) > 0 or page_no == 1:
commit_info = gh.repos(org)(repo_nm).commits.get(
since=str(commit_since), per_page="100", page=str(page_no))
for rec in commit_info:
date = dt.strptime(rec['commit']['author']['date'],
'%Y-%m-%dT%H:%M:%SZ').date()
commit_count_dict[str(date)] = commit_count_dict.get(
str(date), 0) + 1
page_no = page_no + 1
views_14_days = gh.repos(org, repo_nm).traffic.views.get()
count = 0
for view_per_day in views_14_days['views']:
view_dt = dt.strptime(view_per_day['timestamp'],
'%Y-%m-%dT%H:%M:%SZ').date()
if commit_since <= view_dt:
commit_count = commit_count_dict.get(str(view_dt), 0)
git_traffic_rec = [
repo_nm, view_dt, view_per_day['count'],
view_per_day['uniques'], commit_count, forks,
stargazers_count, watchers_count,
dt.utcnow(),
dt.utcnow()
]
rows_inserted = write_to_db(repo,
git_traffic_rec=git_traffic_rec)
count = count + rows_inserted
print("records recorded:", count)
def get_repo_info(request):
if request.method == 'GET':
c = {}
c.update(csrf(request))
return render_to_response("ghactivity/repository_get.html", c)
elif request.method == 'POST':
full_name = request.POST['repository']
owner, name = full_name.split("/", 1)
# try to find this repo in database
existing = Repository.objects.filter(owner=owner, name=name)
if existing.exists():
r = existing.get()
if not r.accessible_by.filter(pk=request.user.pk).exists():
r.accessible_by.add(request.user)
r.save()
return redirect("repo_list")
# get object for accessing github API
token = request.user.social_auth.filter(
provider='github').first().extra_data['access_token']
gh = github.GitHub(access_token=token)
# get basic info - name etc.
try:
rid, full_name, fork, created_at, forked_from = gm.get_basic_repo_info(
gh, owner, name)
except github.ApiNotFoundError:
raise Http404("No such repo exists.")
# save basic info to db
r = Repository(repository_id=rid,
owner=owner,
name=name,
fork=forked_from)
r.save()
r.accessible_by.add(request.user)
r.save()
# start thread for downloading additional info
t = threading.Thread(target=download_repo_info,
args=(gh, owner, name, r))
t.start()
return redirect("repo_list")
def is_user_valid(username, password):
"""
User credentials validator.
Args:
username (str): github username
password (str): github password
Returns:
bool: True if user credentials is valid. False if otherwise
"""
gh = github.GitHub(username=username, password=password)
try:
gh.users('githubpy').followers.get()
return True
except github.ApiError:
return False
def get(self):
if GITHUB_ORG is None or GITHUB_TOKEN is None:
sys.exit(
'You need to provide github org & token: set env GITHUB_ORG, GITHUB_TOKEN'
)
gh = github.GitHub(access_token=GITHUB_TOKEN)
result = []
for item in gh.orgs('gepardec/repos?type=public').get():
try:
views_14_days = gh.repos(GITHUB_ORG,
item.name).traffic.views.get()
for view_per_day in views_14_days['views']:
# Convert String Timestamp to Unix String looks like: 2020-06-18T00:00:00Z
unixtime = time.mktime(
datetime.datetime.strptime(
view_per_day['timestamp'],
'%Y-%m-%dT%H:%M:%SZ').timetuple())
print(
f"INSERT INTO repostats(repo, viewDate, viewCount, uniques) VALUES ('{GITHUB_ORG}/{item.name}', {int(unixtime)}, {view_per_day['count']}, {view_per_day['uniques']})"
)
try:
result.append(
self.executeQueryJson(
f"INSERT INTO repostats(repo, viewDate, viewCount, uniques) OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques VALUES ('{GITHUB_ORG}/{item.name}', {int(unixtime)}, {int(view_per_day['count'])}, {int(view_per_day['uniques'])})"
))
except:
# print(f"UPDATE repostats SET viewCount = {int(view_per_day['count'])}, uniques = {int(view_per_day['uniques'])} OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques WHERE repo = '{GITHUB_ORG}/{item.name}' AND viewDate = {int(unixtime)}")
result.append(
self.executeQueryJson(
f"UPDATE repostats SET viewCount = {int(view_per_day['count'])}, uniques = {int(view_per_day['uniques'])} OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques WHERE repo = '{GITHUB_ORG}/{item.name}' AND viewDate = {int(unixtime)}"
))
timestamp = view_per_day['timestamp']
data = {
'uniques': view_per_day['uniques'],
'count': view_per_day['count']
}
print(item.name, timestamp, data)
except:
print(item.name, "no access", file=sys.stderr)
# result = self.executeQueryJson(f"SELECT ID,habit,occured FROM habits WHERE CONVERT(VARCHAR, habit) = '{habit}' ORDER BY occured DESC")
return result, 200
def collect(db, user, passwd, token, repo, org):
if org is None:
org = user
if token is not None:
user = None
password = None
gh = github.GitHub(username=user, password=passwd, access_token=token)
try:
gh.repos(org, repo).get()
except:
sys.exit('Username/org "' + org + '" or repo "' + repo +
'" not found in github')
if user is not None and org != user:
try:
gh.repos(org, repo).collaborators(user).get()
except:
sys.exit('Username "' + user +
'" does not have collaborator permissions in repo "' +
repo + '"')
views_14_days = gh.repos(org, repo).traffic.views.get()
found_new_data = False
for view_per_day in views_14_days['views']:
timestamp = view_per_day['timestamp']
data = {
'uniques': view_per_day['uniques'],
'count': view_per_day['count']
}
if db.get(timestamp) is None:
db.set(timestamp, json.dumps(data))
print timestamp, data
found_new_data = True
else:
db_data = json.loads(db.get(timestamp))
if db_data['uniques'] < data['uniques']:
db.set(timestamp, json.dumps(data))
print timestamp, data
found_new_data = True
if not found_new_data:
print 'No new traffic data was found for ' + org + '/' + repo
db.dump()
def get_total_insertions_deletions(repo_link, username=None, password=None):
"""
Return the total number of insertions and deletions
API: GET /repos/:owner/:repo/stats/code_frequency
Args:
repo_link (str) : the repository link in the format owner/repo_name
username (str): github username
password (str): github password
Returns:
insertions, deletions
"""
owner, repo = process_repo_link(repo_link)
gh = github.GitHub(username=username,
password=password) if username and password else GITHUB
weekly_data = gh.repos(owner)(repo).stats.code_frequency.get()
adds = 0
dels = 0
for week in weekly_data:
adds += week[1]
dels += week[2]
return adds, dels * -1
def __init__(self):
self.sleep = int(os.environ['SLEEP'])
with open("/opt/service/secret/redis.json", "r") as redis_file:
self.redis = redis.Redis(charset="utf-8",
decode_responses=True,
**json.loads(redis_file.read()))
with open("/opt/service/secret/github.json", "r") as github_file:
self.github = github.GitHub(**json.loads(github_file.read()))
self.cnc = cnc.CnC(self)
subprocess.check_output("mkdir -p /root/.ssh", shell=True)
subprocess.check_output(
"cp /opt/service/secret/github.key /root/.ssh/", shell=True)
subprocess.check_output("chmod 600 /root/.ssh/github.key", shell=True)
subprocess.check_output(
"cp /opt/service/secret/.sshconfig /root/.ssh/config", shell=True)
subprocess.check_output(
"cp /opt/service/secret/.gitconfig /root/.gitconfig", shell=True)
self.env = jinja2.Environment(keep_trailing_newline=True)
def parse_github(temp):
gh = github.GitHub(username=conf['gh_user'], password=conf['gh_pass'])
followers = str(gh.users('coffeecodecouch').get()['followers']).ljust(24)
return temp.replace('{{gh_followers}}', followers)
def main():
if len(sys.argv) < 2:
print("Path to configuration file wasn't specified. Exiting")
exit(1)
config = c.Configuration(sys.argv[1])
repo = r.Repository(config.getRepo(), config)
if repo.checkIfExists() == True:
print("Updating repository " + repo.extractRepositoryName())
repo.Pull()
else:
print("Cloning repository: " + repo.extractRepositoryName())
repo.Clone()
qaRepo = r.Repository(config.getQA(), config)
if config.getRepo() != config.getQA():
if qaRepo.checkIfExists() == True:
print("Updating repository " + qaRepo.extractRepositoryName())
qaRepo.Pull()
else:
print("Cloning repository: " + qaRepo.extractRepositoryName())
qaRepo.Clone()
else:
print("Skipping QA repository: it's the same as test repo")
if not u.CheckRepoPathExists(config, repo, config.getPath()):
print("Configured directory " + config.getPath() +
" wasn't found in test repository. Aborting")
exit(21)
if not u.CheckRepoPathExists(config, qaRepo, config.getQAPath()):
print("Configured directory " + config.getQAPath() +
" wasn't found in test repository. Aborting")
exit(22)
# Workflow starts here
gh = 0
try:
gh = g.GitHub(config.getPrivateKey(), config.getAppID())
gh.Auth()
except ValueError as err:
print("GitHub auth failed: " + str(err))
exit(101)
ghUser = ''
ghOrg = ''
ghType = ''
installation_id = 0
for user in config.getUsers():
installation_id = gh.CheckUserInstallation(user)
ghUser = user
ghType = 'user'
break
for org in config.getOrgs():
installation_id = gh.CheckOrgInstallation(org)
ghOrg = org
ghType = 'org'
break
ghTitle = ''
if ghType == 'user':
ghTitle = ghUser
else:
ghTitle = ghOrg
if installation_id == 0:
print("Couldn't get installation for " + ghType + " " + ghTitle)
exit(210)
installation_id = gh.CheckRepoInstallation('crioto', 'qa-org')
print("Found installation ID for " + ghTitle + ": " + str(installation_id))
gh.AuthInstallation(installation_id)
print(gh.GetIssues('crioto', 'qa-org'))
gh.CreateIssue('crioto', 'qa-org', 'Test Title', 'Test Text', '')
print(gh.GetIssues('crioto', 'qa-org'))
# gh = g.InitializeGithub(config.getToken())
# user = gh.get_user()
# print(user)
exit(0)
builder = b.Builder(
os.path.join(config.getLocalPath(), qaRepo.extractRepositoryName(),
config.getQAPath()))
builder.Run()
issues = builder.Get()
tags = []
for issue in issues:
tags.append(issue.GetAbsoluteHandle())
analyzer = a.Analyzer(
os.path.join(config.getLocalPath(), repo.extractRepositoryName(),
config.getPath()), tags)
analyzer.Run()
covered = analyzer.GetMatches()
import jenkins, github, codeclimate, manage_csv, slack, anomalydetection jen = jenkins.Jenkins() g = github.GitHub() cc = codeclimate.Codeclimate() sl = slack.Slack() csv = manage_csv.Manage_csv(g, cc, jen) ad = anomalydetection.AnomalyDetection(jen, csv, 1) #csv.create_csv() dataset = csv.read_csv() ad.executeAD(dataset['dataset'])
def _agh(self, expUrl, u, t, filename):
def opener(url):
self.assertEquals(expUrl, url + '?login='******'&token=' + t)
return open(filename)
return github.GitHub(fetcher=opener)
def _gh(self, expUrl, filename):
def opener(url):
self.assertEquals(expUrl, url)
return open(filename)
return github.GitHub(fetcher=opener)
