def main(args): username = None password = None fetchGists = False # Parse options opts, _ = getopt.getopt(args, 'gau:', ['gists', 'auth', 'user='******'-a', '--auth'): username = raw_input('Username? ') password = getpass.getpass('Password? ') elif opt in ('-u', '--user'): if ':' in arg: username, password = arg.split(':') else: username = arg elif opt in ('-g', '--gists'): fetchGists = True # Exit if no username set if not username: print 'Please provide a username with -u or ask for a username prompt with -a.' sys.exit(0) # Authenticate if password set if not password: API = github.GitHub() if fetchGists: repos = API.users(username).gists() else: repos = API.users(username).repos() # Anonymous if no password set else: API = github.GitHub(username=username, password=password) if fetchGists: repos = API.gists() else: repos = API.user().repos() # Iterate repos and clone repos = repos.get() for repo in repos: if fetchGists: url = repo.git_pull_url path = repo.id else: url = repo.ssh_url path = repo.name # Don't clone if it already exists in this directory if not os.path.exists(path): _shell('git clone {}'.format(url)) else: print '{} exists, aborting clone'.format(path)
def get_top_contributor_in_past_week(repo_link, username=None, password=None): """ Return top contributor of within the last week Args: repo_link (str) : the repository link in the format owner/repo_name username (str): github username password (str): github password Returns: dict: { 'username': <username>, 'name': <name>, 'commits': <commits> 'additions': <additions> 'deletions': <deletions> } """ gh = github.GitHub(username=username, password=password) if username and password else GITHUB top = _get_contributor_stats(repo_link, gh)[-1] return { 'username': top['author']['login'], 'name': get_name_from_username(top['author']['login']), 'commits': top['weeks'][0]['c'], 'additions': top['weeks'][0]['a'], 'deletions': top['weeks'][0]['d'] }
def get_latest_commit_summary(repo_link, username=None, password=None): """ Extracts latest commit info for a given repository. Args: repo_link (str) : the repository link in the format owner/repo_name username (str): github username password (str): github password Returns: dict: ['user'] --> {'name', 'email', 'username'}, ['message'], ['timestamp'] """ gh = github.GitHub(username=username, password=password) if username and password else GITHUB latest_commit = _get_commits_from_api(repo_link, gh)[0] latest_commit_dict = {} latest_commit_dict['user'] = {} latest_commit_dict['user']['name'] = latest_commit['commit']['author'][ 'name'] latest_commit_dict['user']['email'] = latest_commit['commit']['author'][ 'email'] latest_commit_dict['user']['username'] = latest_commit['committer'][ 'login'] latest_commit_dict['message'] = latest_commit['commit']['message'] latest_commit_dict['timestamp'] = latest_commit['commit']['committer'][ 'date'] return latest_commit_dict
def get_top_n_contributors(repo_link, n, username=None, password=None): """ Extracts top contributors for a given repository. Args: repo_link (str) : the repository link in the format owner/repo_name n (int) : top n contributors; must be greater than 0 Returns: list: top contributors in descending order of contributions the n-th element of the list is the top (n+1)th contributor each element is a dict containing ['username'] and ['contributions'] """ assert n > 0 persons = 0 gh = github.GitHub(username=username, password=password) if username and password else GITHUB contributors = _get_contributors_from_api(repo_link, gh) contributions = [] for contributor in contributors: this_contributor = {} this_contributor['username'] = contributor['login'] this_contributor['contributions'] = contributor['contributions'] contributions.append(this_contributor) persons += 1 # only show top n contributors if persons >= n: break return contributions
def get_stats_by_author(repo_link, author_name, username=None, password=None): """ Return total number of commits, lines added and lines delted by an author Args: repo_link : owner/repo format author_name : limit history to one author username (str) : github username password (str) : github password Return: three numbers: commits, additions, deletions """ owner, repo = process_repo_link(repo_link) gh = github.GitHub(username=username, password=password) if username and password else GITHUB repo_data = gh.repos(owner)(repo).stats.contributors.get() for all_data in repo_data: if all_data['author']['login'] == author_name: total_commits = all_data['total'] weekly_data = all_data['weeks'] adds = 0 dels = 0 for week in weekly_data: adds += week['a'] dels += week['d'] return total_commits, adds, dels
def main(): """Main function. Read the configuration file, instantiate the GitHub object and run through checks""" log = logger.Logger() # Read the organization and Oauth token from the configuration file log.log("Reading configuration") configs = readConfig("config.ini") # Instantiate the github object with the organization and Oauth token from the configuratio file log.log("Instantiating github object") gh = github.GitHub(configs['org'], configs['token']) # Test 1: Check users without 2FA log.log("Running first check: Check users without 2FA enabled") check.check2FA(gh, log) # Test 2: Check for invalid commit signatures log.log("Running second check: Check Invalid Commit Signatures") check.checkCommitSignature(gh, log) # Test 3: Check Installed Webhooks log.log("Running third check: Check installed webhooks") check.checkInstalledWebhooks(gh, log) log.log("Running fourth check: Check team member changes") check.checkTeamMemberChanges(gh, log)
def getgithub(message, CMD, REPOS): # -*- coding: utf-8 -*- import github import sys import os # os.environ でAPI TOKEN 取得 GITHUB_ACCESS_TOKEN = os.environ.get('GITHUB_ACCESS_TOKEN', '') gh = github.GitHub(access_token=GITHUB_ACCESS_TOKEN) # API TOLKEM入力後にusername情報取得 getuser = gh.user.get() USER = getuser['name'] if CMD == "issue": issues = gh.repos(USER)(REPOS).issues().get() print("issues: " + issues[0]['state']) print("issues: " + issues[0]['url']) text = issues[0]['url'] elif CMD == "pull": pulls = gh.repos(USER)(REPOS).pulls().get() print("pulls: " + pulls[0]['review_comments_url']) for key, value in getuser.items(): print(key, value) text = pulls[0]['review_comments_url'] else: text = ("usage:get [issue or pull] [REPOS]") message.send(text)
def is_repo_link_valid(repo_link, username=None, password=None): """ Repository link validator. Args: repo_link (str): the repository link in the format owner/repo_name Returns: bool: True if repository exists (valid owner and repo_name) AND repository is public, False otherwise """ owner = repo = '' try: owner, repo = process_repo_link(repo_link) except ValueError: return False try: gh = github.GitHub( username=username, password=password) if username and password else GITHUB gh.repos(owner)(repo).get() return True except github.ApiNotFoundError: return False
def collect(user, repo, token, org): if org is None: org = user db = __load_db(repo=repo) gh = github.GitHub(access_token=token) try: gh.repos(org, repo).get() except Exception: sys.exit('No Data') if user is not None and org != user: try: gh.repos(org, repo).collaborators(user).get() except Exception: sys.exit('No Data') views_data = gh.repos(org, repo).traffic.views.get() found_new_data = False for view_per_day in views_data['views']: timestamp = view_per_day['timestamp'] data = {'uniques': view_per_day['uniques'], 'count': view_per_day['count']} if db.get(timestamp) is None or db.get(timestamp) is False: db.set(timestamp, json.dumps(data)) print(timestamp, data) found_new_data = True else: db_data = json.loads(db.get(timestamp)) if db_data['uniques'] < data['uniques']: db.set(timestamp, json.dumps(data)) print(timestamp, data) found_new_data = True if not found_new_data: print('No new traffic data was found for ' + org + '/' + repo) db.dump()
def reportIssue(self, username, password, title, body): if username == '' or password == '' or title == '' or body == '': raise ValueError('incorrect parameters,try again.') gh = github.GitHub(username, password) gh.repos('racaljk')('pyann').issues.post(title=title, body=body, label='QuickIssue') print 'You have succeed to report a Issue to hosts project via getHost script.'
def get_repo_contributors(repo_link, username=None, password=None): gh = github.GitHub(username=username, password=password) if username and password else GITHUB contributors = _get_contributors_from_api(repo_link, gh) contributor_list = [] for contributor in contributors: contributor_list.append(contributor['login']) return contributor_list
def test___init__(self): init = github.GitHub("me", "arcade", "git.com") self.assertEqual(init.user, "me") self.assertEqual(init.token, "arcade") self.assertEqual(init.api.auth, ("me", "arcade")) self.assertEqual(init.url, "git.com")
def _ghp(self, expUrl, u, t, **kv): def opener(url, data): h = {'login': u, 'token': t} h.update(kv) self.assertEquals(github.BaseEndpoint.BASE_URL + expUrl, url) self.assertEquals(sorted(data.split('&')), sorted(urllib.urlencode(h).split('&'))) return github.GitHub(u, t, fetcher=opener)
def github_client(): """ A fixture to initialize a GitHub API client, using the personal access token obtained from the GITHUB_TOKEN environment variable. """ try: token = os.environ["GITHUB_TOKEN"] except KeyError: raise Exception("To use any of the GitHub fixtures, you must set the GITHUB_TOKEN environment variable " "to contain a GitHub personal access token.") return github.GitHub(token)
def main(token, repo, db): logging.basicConfig(level=logging.INFO) hub = github.GitHub(access_token=token) with sqlite3.connect(db) as rdb: rdb.row_factory = row_factory for loader in LOADERS: rdb.cursor().execute(loader.schema) for r in repo: loader(hub, rdb, r)
def update_repository(request, pk): r = get_object_or_404(request.user.access, pk=pk) token = request.user.social_auth.filter( provider='github').first().extra_data['access_token'] gh = github.GitHub(access_token=token) # stat thread for downloading new info t = threading.Thread(target=download_repo_info, args=(gh, r.owner, r.name, r, True)) t.start() return redirect("repo_list")
def authenticate(self): """ (class) -> boolean Passing throught authentication and verifying if the right credentials were given. """ if self.personal_access_token is not None: self.gh = github.GitHub(access_token = self.personal_access_token) else: self.gh = github.GitHub( self.consumer_key, self.consumer_secret, self.access_token, self.access_secret) try: some_get = self.gh.users(self.user_name).get() print json.dumps(some_get, sort_keys=True, indent=4, separators=(',', ': ')) return True except Exception, ex: print '[e] exception {}'.format(str(ex)) return False
def collect(user, repo, org): """ Function for Collection of Data. Counts the total number of views (Unique and total) on a Repo. Parameters: user (str): The Github Username repo (str): The Name of Repo org (str): The Name of Organization [Optional] """ token = decouple.config("API_KEY") if org is None: org = user db = __load_db(repo=repo) gh = github.GitHub(access_token=token) try: gh.repos(org, repo).get() except Exception: sys.exit("No Data") if user is not None and org != user: try: gh.repos(org, repo).collaborators(user).get() except Exception: sys.exit("No Data") views_data = gh.repos(org, repo).traffic.views.get() found_new_data = False for view_per_day in views_data["views"]: timestamp = view_per_day["timestamp"] data = { "uniques": view_per_day["uniques"], "count": view_per_day["count"] } if db.get(timestamp) is None or db.get(timestamp) is False: db.set(timestamp, json.dumps(data)) print(timestamp, data) found_new_data = True else: db_data = json.loads(db.get(timestamp)) if db_data["uniques"] < data["uniques"]: db.set(timestamp, json.dumps(data)) print(timestamp, data) found_new_data = True if not found_new_data: print("No new traffic data was found for " + org + "/" + repo) db.dump()
def _collect(token, org, repo): gh = github.GitHub(access_token=token) repositries_data = gh.orgs(org).repos.get(per_page="1000") print("total no. of public repository: ", len(repositries_data)) for rec in repositries_data: repo_nm = rec['name'] forks = rec['forks'] stargazers_count = rec['stargazers_count'] watchers_count = rec['watchers_count'] print("processing repo: ", repo_nm) commit_since = last_recorded_date(repo, repo_nm) if commit_since == None: commit_since = (dt.utcnow() - td(14)).date() page_no = 1 commit_info = {} commit_count_dict = {} while len(commit_info) > 0 or page_no == 1: commit_info = gh.repos(org)(repo_nm).commits.get( since=str(commit_since), per_page="100", page=str(page_no)) for rec in commit_info: date = dt.strptime(rec['commit']['author']['date'], '%Y-%m-%dT%H:%M:%SZ').date() commit_count_dict[str(date)] = commit_count_dict.get( str(date), 0) + 1 page_no = page_no + 1 views_14_days = gh.repos(org, repo_nm).traffic.views.get() count = 0 for view_per_day in views_14_days['views']: view_dt = dt.strptime(view_per_day['timestamp'], '%Y-%m-%dT%H:%M:%SZ').date() if commit_since <= view_dt: commit_count = commit_count_dict.get(str(view_dt), 0) git_traffic_rec = [ repo_nm, view_dt, view_per_day['count'], view_per_day['uniques'], commit_count, forks, stargazers_count, watchers_count, dt.utcnow(), dt.utcnow() ] rows_inserted = write_to_db(repo, git_traffic_rec=git_traffic_rec) count = count + rows_inserted print("records recorded:", count)
def get_repo_info(request): if request.method == 'GET': c = {} c.update(csrf(request)) return render_to_response("ghactivity/repository_get.html", c) elif request.method == 'POST': full_name = request.POST['repository'] owner, name = full_name.split("/", 1) # try to find this repo in database existing = Repository.objects.filter(owner=owner, name=name) if existing.exists(): r = existing.get() if not r.accessible_by.filter(pk=request.user.pk).exists(): r.accessible_by.add(request.user) r.save() return redirect("repo_list") # get object for accessing github API token = request.user.social_auth.filter( provider='github').first().extra_data['access_token'] gh = github.GitHub(access_token=token) # get basic info - name etc. try: rid, full_name, fork, created_at, forked_from = gm.get_basic_repo_info( gh, owner, name) except github.ApiNotFoundError: raise Http404("No such repo exists.") # save basic info to db r = Repository(repository_id=rid, owner=owner, name=name, fork=forked_from) r.save() r.accessible_by.add(request.user) r.save() # start thread for downloading additional info t = threading.Thread(target=download_repo_info, args=(gh, owner, name, r)) t.start() return redirect("repo_list")
def is_user_valid(username, password): """ User credentials validator. Args: username (str): github username password (str): github password Returns: bool: True if user credentials is valid. False if otherwise """ gh = github.GitHub(username=username, password=password) try: gh.users('githubpy').followers.get() return True except github.ApiError: return False
def get(self): if GITHUB_ORG is None or GITHUB_TOKEN is None: sys.exit( 'You need to provide github org & token: set env GITHUB_ORG, GITHUB_TOKEN' ) gh = github.GitHub(access_token=GITHUB_TOKEN) result = [] for item in gh.orgs('gepardec/repos?type=public').get(): try: views_14_days = gh.repos(GITHUB_ORG, item.name).traffic.views.get() for view_per_day in views_14_days['views']: # Convert String Timestamp to Unix String looks like: 2020-06-18T00:00:00Z unixtime = time.mktime( datetime.datetime.strptime( view_per_day['timestamp'], '%Y-%m-%dT%H:%M:%SZ').timetuple()) print( f"INSERT INTO repostats(repo, viewDate, viewCount, uniques) VALUES ('{GITHUB_ORG}/{item.name}', {int(unixtime)}, {view_per_day['count']}, {view_per_day['uniques']})" ) try: result.append( self.executeQueryJson( f"INSERT INTO repostats(repo, viewDate, viewCount, uniques) OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques VALUES ('{GITHUB_ORG}/{item.name}', {int(unixtime)}, {int(view_per_day['count'])}, {int(view_per_day['uniques'])})" )) except: # print(f"UPDATE repostats SET viewCount = {int(view_per_day['count'])}, uniques = {int(view_per_day['uniques'])} OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques WHERE repo = '{GITHUB_ORG}/{item.name}' AND viewDate = {int(unixtime)}") result.append( self.executeQueryJson( f"UPDATE repostats SET viewCount = {int(view_per_day['count'])}, uniques = {int(view_per_day['uniques'])} OUTPUT INSERTED.repo,INSERTED.viewDate,INSERTED.viewCount,INSERTED.uniques WHERE repo = '{GITHUB_ORG}/{item.name}' AND viewDate = {int(unixtime)}" )) timestamp = view_per_day['timestamp'] data = { 'uniques': view_per_day['uniques'], 'count': view_per_day['count'] } print(item.name, timestamp, data) except: print(item.name, "no access", file=sys.stderr) # result = self.executeQueryJson(f"SELECT ID,habit,occured FROM habits WHERE CONVERT(VARCHAR, habit) = '{habit}' ORDER BY occured DESC") return result, 200
def collect(db, user, passwd, token, repo, org): if org is None: org = user if token is not None: user = None password = None gh = github.GitHub(username=user, password=passwd, access_token=token) try: gh.repos(org, repo).get() except: sys.exit('Username/org "' + org + '" or repo "' + repo + '" not found in github') if user is not None and org != user: try: gh.repos(org, repo).collaborators(user).get() except: sys.exit('Username "' + user + '" does not have collaborator permissions in repo "' + repo + '"') views_14_days = gh.repos(org, repo).traffic.views.get() found_new_data = False for view_per_day in views_14_days['views']: timestamp = view_per_day['timestamp'] data = { 'uniques': view_per_day['uniques'], 'count': view_per_day['count'] } if db.get(timestamp) is None: db.set(timestamp, json.dumps(data)) print timestamp, data found_new_data = True else: db_data = json.loads(db.get(timestamp)) if db_data['uniques'] < data['uniques']: db.set(timestamp, json.dumps(data)) print timestamp, data found_new_data = True if not found_new_data: print 'No new traffic data was found for ' + org + '/' + repo db.dump()
def get_total_insertions_deletions(repo_link, username=None, password=None): """ Return the total number of insertions and deletions API: GET /repos/:owner/:repo/stats/code_frequency Args: repo_link (str) : the repository link in the format owner/repo_name username (str): github username password (str): github password Returns: insertions, deletions """ owner, repo = process_repo_link(repo_link) gh = github.GitHub(username=username, password=password) if username and password else GITHUB weekly_data = gh.repos(owner)(repo).stats.code_frequency.get() adds = 0 dels = 0 for week in weekly_data: adds += week[1] dels += week[2] return adds, dels * -1
def __init__(self): self.sleep = int(os.environ['SLEEP']) with open("/opt/service/secret/redis.json", "r") as redis_file: self.redis = redis.Redis(charset="utf-8", decode_responses=True, **json.loads(redis_file.read())) with open("/opt/service/secret/github.json", "r") as github_file: self.github = github.GitHub(**json.loads(github_file.read())) self.cnc = cnc.CnC(self) subprocess.check_output("mkdir -p /root/.ssh", shell=True) subprocess.check_output( "cp /opt/service/secret/github.key /root/.ssh/", shell=True) subprocess.check_output("chmod 600 /root/.ssh/github.key", shell=True) subprocess.check_output( "cp /opt/service/secret/.sshconfig /root/.ssh/config", shell=True) subprocess.check_output( "cp /opt/service/secret/.gitconfig /root/.gitconfig", shell=True) self.env = jinja2.Environment(keep_trailing_newline=True)
def parse_github(temp): gh = github.GitHub(username=conf['gh_user'], password=conf['gh_pass']) followers = str(gh.users('coffeecodecouch').get()['followers']).ljust(24) return temp.replace('{{gh_followers}}', followers)
def main(): if len(sys.argv) < 2: print("Path to configuration file wasn't specified. Exiting") exit(1) config = c.Configuration(sys.argv[1]) repo = r.Repository(config.getRepo(), config) if repo.checkIfExists() == True: print("Updating repository " + repo.extractRepositoryName()) repo.Pull() else: print("Cloning repository: " + repo.extractRepositoryName()) repo.Clone() qaRepo = r.Repository(config.getQA(), config) if config.getRepo() != config.getQA(): if qaRepo.checkIfExists() == True: print("Updating repository " + qaRepo.extractRepositoryName()) qaRepo.Pull() else: print("Cloning repository: " + qaRepo.extractRepositoryName()) qaRepo.Clone() else: print("Skipping QA repository: it's the same as test repo") if not u.CheckRepoPathExists(config, repo, config.getPath()): print("Configured directory " + config.getPath() + " wasn't found in test repository. Aborting") exit(21) if not u.CheckRepoPathExists(config, qaRepo, config.getQAPath()): print("Configured directory " + config.getQAPath() + " wasn't found in test repository. Aborting") exit(22) # Workflow starts here gh = 0 try: gh = g.GitHub(config.getPrivateKey(), config.getAppID()) gh.Auth() except ValueError as err: print("GitHub auth failed: " + str(err)) exit(101) ghUser = '' ghOrg = '' ghType = '' installation_id = 0 for user in config.getUsers(): installation_id = gh.CheckUserInstallation(user) ghUser = user ghType = 'user' break for org in config.getOrgs(): installation_id = gh.CheckOrgInstallation(org) ghOrg = org ghType = 'org' break ghTitle = '' if ghType == 'user': ghTitle = ghUser else: ghTitle = ghOrg if installation_id == 0: print("Couldn't get installation for " + ghType + " " + ghTitle) exit(210) installation_id = gh.CheckRepoInstallation('crioto', 'qa-org') print("Found installation ID for " + ghTitle + ": " + str(installation_id)) gh.AuthInstallation(installation_id) print(gh.GetIssues('crioto', 'qa-org')) gh.CreateIssue('crioto', 'qa-org', 'Test Title', 'Test Text', '') print(gh.GetIssues('crioto', 'qa-org')) # gh = g.InitializeGithub(config.getToken()) # user = gh.get_user() # print(user) exit(0) builder = b.Builder( os.path.join(config.getLocalPath(), qaRepo.extractRepositoryName(), config.getQAPath())) builder.Run() issues = builder.Get() tags = [] for issue in issues: tags.append(issue.GetAbsoluteHandle()) analyzer = a.Analyzer( os.path.join(config.getLocalPath(), repo.extractRepositoryName(), config.getPath()), tags) analyzer.Run() covered = analyzer.GetMatches()
import jenkins, github, codeclimate, manage_csv, slack, anomalydetection jen = jenkins.Jenkins() g = github.GitHub() cc = codeclimate.Codeclimate() sl = slack.Slack() csv = manage_csv.Manage_csv(g, cc, jen) ad = anomalydetection.AnomalyDetection(jen, csv, 1) #csv.create_csv() dataset = csv.read_csv() ad.executeAD(dataset['dataset'])
def _agh(self, expUrl, u, t, filename): def opener(url): self.assertEquals(expUrl, url + '?login='******'&token=' + t) return open(filename) return github.GitHub(fetcher=opener)
def _gh(self, expUrl, filename): def opener(url): self.assertEquals(expUrl, url) return open(filename) return github.GitHub(fetcher=opener)