def get_merge_request_overview() -> PullRequestsOverview: _prs_to_review: List[PullRequest] = [] _prs_authored_with_work: List[PullRequest] = [] _exception = None _gl = Gitlab(url=GitlabMrsConfig.GITLAB_HOST, private_token=GitlabMrsConfig.PRIVATE_TOKEN) _gl.auth() _author_id = _gl.user.id mrs = group_mrs(_gl) try: _prs_to_review: List[PullRequest] = extract_pull_request_data( get_merge_requests_to_review(_author_id, mrs)) _prs_authored_with_work: List[PullRequest] = extract_pull_request_data( get_authored_merge_requests(_author_id, mrs)) except Timeout as e: _exception = PullRequestException(GitlabMrsConstants.MODULE, GitlabMrsConstants.TIMEOUT_MESSAGE, e, traceback.format_exc()) except GitlabHttpError as e: _exception = PullRequestException( GitlabMrsConstants.MODULE, GitlabMrsConstants.CONNECTION_MESSAGE, e, traceback.format_exc()) except Exception as e: _exception = PullRequestException(GitlabMrsConstants.MODULE, GitlabMrsConstants.UNKNOWN_MESSAGE, e, traceback.format_exc()) return PullRequestsOverview.create(_prs_to_review, _prs_authored_with_work, _exception)
def connect_api(self, max_tries: int = 3): token = self.token # user gets max_tries tries to get the token correct tries = 0 success = False gl = None while not success and tries < max_tries: try: if not (isinstance(token, str) and len(token) > 0): raise Exception('Invalid token') gl = Gitlab(url=self.remote_host, private_token=token) # type: Gitlab gl.auth() success = True except Exception: tries += 1 token = self.set_token() if not success: print('Could not validate ' + self.TOKEN_NAME + '.') quit() self.api = gl
def process_step_4(message): cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))}) cur = [] cursor4 = dict(cursor3) for j in cursor4["token"]: cur.append(decoder(j)) if message.text in cur: try: gl = Gitlab('https://git.iu7.bmstu.ru/', private_token=message.text) gl.auth() username = gl.user.username db.token.find_one_and_update( { "id": encoder(str(message.chat.id)), "token": encoder(cur) }, {'$set': { "idGitLab": encoder(username) }}) except gitlab.GitlabAuthenticationError: st = open('./static/access_denied.webp', 'rb') bot.send_sticker(message.chat.id, st) bot.send_message( message.chat.id, "Произошла ошибка при авторизации в GitLab. Проверьте правильность токена", parse_mode="html", reply_markup=types.ReplyKeyboardRemove()) else: bot.send_message(message.chat.id, "Такого TOKEN нет в твоем списке...", parse_mode="html", reply_markup=types.ReplyKeyboardRemove())
def main(args): repoName = str(args.repopath.stem) repoOwner = str(Path(str(args.repopath.parent).replace(':', '/')).stem) config = loadConfig(args.config) assert config, "could not load configuation at {}".format(str(args.config)) # Gitlab or custom hostname gl = Gitlab("https://{}".format(config['hostname']), private_token=args.token) gl.auth() ownerId, isUser = getGitlabOwnerId(gl, args.repopath) if not ownerId: print("could not find project owner", repoOwner) if isUser: if ownerId and repoName not in getGitlabUserRepoNames(gl, ownerId): createGitlabUserRepo(gl, config, ownerId, repoName, args.repodesc) else: print(args.repopath, "already exists") else: if ownerId and repoName not in getGitlabUserGroupNames(gl, ownerId): createGitlabGroupRepo(gl, config, ownerId, repoName, args.repodesc) else: print(args.repopath, "already exists")
def clone(self, repo_name, repo_dst: Path): gl = Gitlab(url=self.url, private_token=self.auth_token) gl.auth() current_user = gl.user username = current_user.username https_link = f'https://{username}:{self.auth_token}@' + self.url[ 8:] + f"/{username}/{repo_name}.git" return git.Repo.clone_from(https_link, str(repo_dst))
def main(): pipeline_id = int(os.environ["CI_PIPELINE_ID"]) project_id = os.environ["CI_PROJECT_ID"] branch = os.environ["CI_COMMIT_REF_NAME"] _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"]) ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc) gitlab_token = os.environ["GITLAB_TOKEN"] gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4) gitlab_client.auth() project = gitlab_client.projects.get(project_id) interesting_status = ["running", "pending"] futures = [] with ThreadPoolExecutor(max_workers=2) as executor: for status in interesting_status: futures.append( executor.submit(project.pipelines.list, order_by="id", sort="desc", per_page=1, ref=branch, status=status)) latest_pipelines = [] for future in futures: latest_pipelines += future.result() latest_pipelines = sorted(latest_pipelines, key=lambda key: key.attributes["id"], reverse=True) if len(latest_pipelines) == 0: print( "No running pipelines (has a single job been retried?) - continuing" ) exit(0) else: latest_pipeline = latest_pipelines[0] assert pipeline_id <= latest_pipeline.id if pipeline_id < latest_pipeline.id: print( "Running pipeline (%s) is not the latest (%s) - cancelling self" % (pipeline_id, latest_pipeline.id), flush=True) pipeline = project.pipelines.get(pipeline_id, lazy=True) pipeline.cancel() # Call does not block until actioned so going into a sleep from which it won't wake up... sleep(999) assert False else: print("Running pipeline (%s) is the latest - continuing" % (pipeline_id, )) exit(0)
def gitlab_userdata(tokens): gh = Gitlab("https://gitlab.com", oauth_token=tokens["access_token"]) gh.auth() gluser = gh.user return { "id": gluser.id, "login": gluser.username, "name": gluser.name, "avatar_url": gluser.avatar_url, }
def get_clone_commands(token, repo_root): con = Gitlab("http://gitlab.your.domain", token) con.auth() for project in con.Project(per_page=200): url = project.ssh_url_to_repo subdir = url.split(":")[1].split("/")[0] cmd = "bash -c '(mkdir -p {repo_root}/{subdir} && cd {repo_root}/{subdir} && git clone {url})'".format( **locals()) yield cmd
def is_ci_job_running(job_id: int): project_id = os.environ["CI_PROJECT_ID"] _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"]) ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc) gitlab_token = os.environ["GITLAB_TOKEN"] gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4) gitlab_client.auth() project = gitlab_client.projects.get(project_id) job = project.jobs.get(job_id) return job.attributes["status"] == "running"
def is_ci_job_running(job_id: int): project_id = os.environ["CI_PROJECT_ID"] _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"]) ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc) gitlab_token = os.environ["GITLAB_TOKEN"] gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4) gitlab_client.auth() project = gitlab_client.projects.get(project_id) job = project.jobs.get(job_id) return job.attributes["status"] == "running"
def gitlab_connect(token=None): """Authenticate the GL wrapper with Gitlab. Args: token (str): The Gitlab token to authenticate with. Defaults to: None. """ token = settings.GITLAB_API_TOKEN try: gitlab_client = Gitlab("https://gitlab.com", private_token=token) gitlab_client.auth() except GitlabHttpError as e: logger.exception(e) return gitlab_client
def get_gl_repos(owner, token): gl = Gitlab(url='https://gitlab.com', private_token=token) gl.auth() users = gl.users.list(username=owner) if len(users) > 0: user = users[0] else: user = gl.groups.get(owner) repos = user.projects.list(visibility='public') git_urls = list() gl_urls = list() for repo in repos: git_urls.append(repo.http_url_to_repo) gl_urls.append(repo.web_url) return gl_urls, git_urls
async def server_login(self, evt: MessageEvent, url: str, token: str) -> None: gl = Gl(url, private_token=token) try: gl.auth() except GitlabAuthenticationError: await evt.reply("Invalid access token") return except Exception as e: self.log.warning( f"Unexpected error logging into GitLab {url} for {evt.sender}", exc_info=True) await evt.reply(f"GitLab login failed: {e}") return self.db.add_login(evt.sender, url, token) await evt.reply( f"Successfully logged into GitLab at {url} as {gl.user.name}")
def process_step_2(message): cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))}) cur = [] cursor4 = dict(cursor3) for j in cursor4["token"]: cur.append(decoder(j)) if message.text in cur: bot.send_message(message.chat.id, "Данный TOKEN уже есть в нашей базе данных", parse_mode="html", reply_markup=types.ReplyKeyboardRemove()) else: cur.append(message.text) db.token.find_one_and_update({"id": encoder(str(message.chat.id))}, {'$set': { "token": encoder(cur) }}) bot.send_message(message.chat.id, "Ваш TOKEN был успешно добавлен в нашу базу данных 🎉", parse_mode="html", reply_markup=types.ReplyKeyboardRemove()) try: gl = Gitlab('https://git.iu7.bmstu.ru/', private_token=message.text) gl.auth() username = gl.user.username db.token.find_one_and_update( { "id": encoder(str(message.chat.id)), "token": encoder(cur) }, {'$set': { "idGitLab": encoder(username) }}) except gitlab.GitlabAuthenticationError: st4 = open('./static/access_denied.webp', 'rb') bot.send_sticker(message.chat.id, st4) bot.send_message( message.chat.id, "Произошла ошибка при авторизации в GitLab. Проверьте правильность токена", parse_mode="html", reply_markup=types.ReplyKeyboardRemove())
def setUpClass(cls): cls._gitlab_controller = GitLab8_16_6_ce_0ServiceController() cls._gitlab_service = cls._gitlab_controller.start_service() cls.gitlab_location = f"http://{cls._gitlab_service.host}:{cls._gitlab_service.ports[_GITLAB_PORT]}" gitlab = Gitlab(url=cls.gitlab_location, email=cls._gitlab_service.root_user.username, password=cls._gitlab_service.root_user.password) gitlab.auth() cls.gitlab_token = gitlab.private_token cls.project_name = f"{cls._gitlab_service.root_user.username}/{EXAMPLE_PROJECT}" gitlab.projects.create({"name": EXAMPLE_PROJECT}) project_variables_manager = ProjectVariablesManager( GitLabConfig(cls.gitlab_location, cls.gitlab_token), cls.project_name) project_variables_manager.set(EXAMPLE_VARIABLES)
def crear_repo(package_name): md5hash_pn = md5(package_name).hexdigest() first_pref = md5hash_pn[0:2] second_pref = md5hash_pn[2:4] workingdir = root_git_dir+"/"+first_pref+"/"+second_pref+"/" + package_name if not(os.access(root_git_dir+"/"+first_pref, os.F_OK)): os.mkdir(root_git_dir+"/"+first_pref) os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref) elif not(os.access(root_git_dir+"/"+first_pref+"/"+second_pref, os.F_OK)): os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref) repo = pygit2.init_repository(workingdir) dashed_package_name=package_name.replace('.','-').lower() myRemote = repo.remotes.create(package_name, gitlab_url+'/marvin/'+dashed_package_name+'.git') gl = Gitlab (gitlab_url, gitlab_token) gl.auth() p = gl.Project({'name': package_name, 'public':True}) p.save() return repo
def check_gitlab_repo_for_changes(pkg_obj, build_pkgs): gl = Gitlab('https://gitlab.com', GITLAB_TOKEN) gl.auth() project_id = pkg_obj.mon_project repo = pkg_obj.mon_repo project = gl.projects.get(project_id) last_result = pkg_obj.mon_last_result events = project.events.list() for event in events: if event.action_name == 'pushed to': if event.created_at != last_result: pkg_obj.mon_last_result = event.created_at build_pkgs.append('numix-icon-theme-square') break return build_pkgs
def GitLab_Work_Log(url, token): gl = Gitlab(url, token) gl.auth() work_log = '' for project in gl.Project(): work_log += "Project Name: " + project.name + '\n' for commit in project.Commit(): if commit.author_email == gl.user.email: commited_at = datetime.strptime(commit.created_at[:19], '%Y-%m-%dT%H:%M:%S') if datetime.today().date()-timedelta(1) == commited_at.date(): work_log += commit.message + '\n' for merge in project.MergeRequest(): if merge.author.id == gl.user.id: merged_at = datetime.strptime(merge.created_at, '%Y-%m-%dT%H:%M:%S.%fZ') if datetime.today().date()-timedelta(1) == merged_at.date(): work_log += merge.title + '\n' return work_log
class GitLabModule(NotifyModule): """A module to follow gitlab's projects""" _config = (("url", str, ""), ("token", str, ""), ("ssl_verify", bool, True), ("max_commits", int, 10)) def __init__(self, bot): NotifyModule.__init__(self, bot, name="gitlab", desc="Gitlab Interface", delay=60) self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify) self.gl.auth() # DEBUG p = GitLabProject(id=31) pr = self.gl.projects.get(31) p.update_from_gitlab(pr) self.bot.session.add(p) self.bot.session.commit() def do_action(self): issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()} for prj in self.bot.session.query(GitLabProject).all(): gl_prj = self.gl.projects.get(prj.id) commits = gl_prj.commits.list(per_page=self.max_commits) last = commits[0].id while commits and commits[0].id != prj.last_commit: commit = commits.pop(0) self.bot.say(MSG['commit'] % (prj.name, commit.author_name, commit.title)) prj.last_commit = last for issue in gl_prj.issues.list(all=True): if issue.id not in issues.keys(): issues[issue.id] = GitLabIssue(id=issue.id) self.bot.session.add(issues[issue.id]) if issue.state == 'opened': self.bot.say( MSG['issue'] % (prj.name, issue.title, prj.web_url, issue.iid)) issues[issue.id].update_from_gitlab(issue) self.bot.session.commit()
class GitLabData: """GitLab Data object.""" def __init__(self, gitlab_id, priv_token, interval, url): """Fetch data from GitLab API for most recent CI job.""" self._gitlab_id = gitlab_id self._gitlab = Gitlab(url, private_token=priv_token, per_page=1) self._gitlab.auth() self.update = Throttle(interval)(self._update) self.available = False self.status = None self.started_at = None self.finished_at = None self.duration = None self.commit_id = None self.commit_date = None self.build_id = None self.branch = None def _update(self): try: _projects = self._gitlab.projects.get(self._gitlab_id) _last_pipeline = _projects.pipelines.list(page=1)[0] _last_job = _last_pipeline.jobs.list(page=1)[0] self.status = _last_pipeline.attributes.get("status") self.started_at = _last_job.attributes.get("started_at") self.finished_at = _last_job.attributes.get("finished_at") self.duration = _last_job.attributes.get("duration") _commit = _last_job.attributes.get("commit") self.commit_id = _commit.get("id") self.commit_date = _commit.get("committed_date") self.build_id = _last_job.attributes.get("id") self.branch = _last_job.attributes.get("ref") self.available = True except GitlabAuthenticationError as erra: _LOGGER.error("Authentication Error: %s", erra) self.available = False except GitlabGetError as errg: _LOGGER.error("Project Not Found: %s", errg) self.available = False
def crear_repo(package_name): md5hash_pn = md5(package_name).hexdigest() first_pref = md5hash_pn[0:2] second_pref = md5hash_pn[2:4] workingdir = root_git_dir + "/" + first_pref + "/" + second_pref + "/" + package_name if not (os.access(root_git_dir + "/" + first_pref, os.F_OK)): os.mkdir(root_git_dir + "/" + first_pref) os.mkdir(root_git_dir + "/" + first_pref + "/" + second_pref) elif not (os.access(root_git_dir + "/" + first_pref + "/" + second_pref, os.F_OK)): os.mkdir(root_git_dir + "/" + first_pref + "/" + second_pref) repo = pygit2.init_repository(workingdir) dashed_package_name = package_name.replace('.', '-').lower() myRemote = repo.remotes.create( package_name, gitlab_url + '/marvin/' + dashed_package_name + '.git') gl = Gitlab(gitlab_url, gitlab_token) gl.auth() p = gl.Project({'name': package_name, 'public': True}) p.save() return repo
def create_gitlab_project( default_has_issues, default_has_wiki, gitlab_secure_config, options, project, description, homepage): created = False has_issues = 'has-issues' in options or default_has_issues has_wiki = 'has-wiki' in options or default_has_wiki secure_config = ConfigParser.ConfigParser() secure_config.read(gitlab_secure_config) # Project creation doesn't work via oauth glab = Gitlab(secure_config.get("gitlab", "url"), secure_config.get("gitlab", "key")) glab.auth() orgs = glab.Group() orgs_dict = dict(zip([o.name.lower() for o in orgs], orgs)) # Find the project's repo project_split = project.split('/', 1) org_name = project_split[0] if len(project_split) > 1: repo_name = project_split[1] else: repo_name = project org_name = 'ustack' try: org = orgs_dict[org_name.lower()] except Exception: # we do not have control of this github org ignore the project. return False if glab.search_projects(repo_name): return created project_info = {'name': repo_name, 'namespace_id': org.id, 'wiki_enabled': has_wiki, 'description': description, 'issues_enabled': has_issues} glab.Project(project_info).save() created = True return created
class GitlabProject: def __init__(self, gitlab_url, api_token, namespace, project_slug): self.gitlab_url = gitlab_url self.api_token = api_token self.namespace = namespace self.project_slug = project_slug self.client = Gitlab(self.gitlab_url, self.api_token, ssl_verify=False) @property def url(self): return "{}/{}/{}".format(self.gitlab_url, self.namespace, self.project_slug) def exists(self): return len(self.client.projects.list(search=self.project_slug)) > 0 def create(self): grp = self.client.groups.list(search=self.namespace)[0] proj_metadata = {'name': self.project_slug, 'namespace_id': grp.id} proj = self.client.projects.create(proj_metadata, retry_transient_errors=True) return proj def add_issue(self, opts): project = self.get_project() user = self.get_current_user() opts['assignee_id'] = user.id return project.issues.create(opts) def get_project(self): try: return self.project except AttributeError: arg = "{}/{}".format(self.namespace, self.project_slug) self.project = self.client.projects.get(arg) return self.project def get_current_user(self): self.client.auth() return self.client.user
def borrar_repo(package_name): filepath = repo_name(package_name) gl = Gitlab (gitlab_url, gitlab_token) gl.auth() # Project search no anda bien, da error si pongo el package_name entero # Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar # asi que buscamos por la ultima palabra del nombre (esperando que no sea Android) # e iteramos sobre los resultados hasta encontrar package_name split_name = package_name.split('.') searchterm = split_name[len(split_name)-1] projlist = gl.search_projects(searchterm) for project in projlist: if project.name == package_name: break else: project = None if project == None: raise Exception("El proyecto no existe en GitLab") else: project.delete() rmtree(filepath)
def borrar_repo(package_name): filepath = repo_name(package_name) gl = Gitlab(gitlab_url, gitlab_token) gl.auth() # Project search no anda bien, da error si pongo el package_name entero # Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar # asi que buscamos por la ultima palabra del nombre (esperando que no sea Android) # e iteramos sobre los resultados hasta encontrar package_name split_name = package_name.split('.') searchterm = split_name[len(split_name) - 1] projlist = gl.search_projects(searchterm) for project in projlist: if project.name == package_name: break else: project = None if project == None: raise Exception("El proyecto no existe en GitLab") else: project.delete() rmtree(filepath)
class GitLabModule(NotifyModule): """A module to follow gitlab's projects""" _config = (("url", str, ""), ("token", str, ""), ("ssl_verify", bool, True), ("max_commits", int, 10)) def __init__(self, bot): NotifyModule.__init__(self, bot, name="gitlab", desc="Gitlab Interface", delay=60) self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify) self.gl.auth() # DEBUG p = GitLabProject(id=31) pr = self.gl.projects.get(31) p.update_from_gitlab(pr) self.bot.session.add(p) self.bot.session.commit() def do_action(self): issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()} for prj in self.bot.session.query(GitLabProject).all(): gl_prj = self.gl.projects.get(prj.id) commits = gl_prj.commits.list(per_page=self.max_commits) last = commits[0].id while commits and commits[0].id != prj.last_commit: commit = commits.pop(0) self.bot.say(MSG['commit'] % (prj.name, commit.author_name, commit.title)) prj.last_commit = last for issue in gl_prj.issues.list(all=True): if issue.id not in issues.keys(): issues[issue.id] = GitLabIssue(id=issue.id) self.bot.session.add(issues[issue.id]) if issue.state == 'opened': self.bot.say(MSG['issue'] % (prj.name, issue.title, prj.web_url, issue.iid)) issues[issue.id].update_from_gitlab(issue) self.bot.session.commit()
def auth(**kwargs): ''' Set up gitlab authenticated client ''' prefix = "gitlab." # look in kwargs first, then default to config file def get_key(key, default=None): return kwargs.get('connection_' + key, __salt__['config.get'](prefix + key, default)) user = get_key('user', 'admin') password = get_key('password', 'ADMIN') token = get_key('token') url = get_key('url', 'https://localhost/') LOG.info("Making HTTP request to {0} ...".format(url)) if token: git = Gitlab(url, token, ssl_verify=False) else: git = Gitlab(url, email=user, password=password, ssl_verify=False) git.auth() return git
def check_gitlab_repo(project_id=None): """ :param project_id: :return: """ new_items = [] gl = Gitlab('https://gitlab.com', GITLAB_TOKEN) gl.auth() nxsq = gl.projects.get(project_id) key = 'antbs:monitor:gitlab:{0}'.format(project_id) last_updated = db.get(key) events = nxsq.events.list() for event in events: if event.action_name == 'pushed to': if event.created_at != last_updated: db.set(key, event.created_at) new_items = ['numix-icon-theme-square'] break return new_items
def check_gitlab_repo(project_id=None): """ :param project_id: :return: """ new_items = [] gl = Gitlab("https://gitlab.com", GITLAB_TOKEN) gl.auth() nxsq = gl.Project(id=project_id) key = "antbs:monitor:gitlab:%s" % project_id last_updated = db.get(key) events = nxsq.Event() for event in events: if event.action_name == "pushed to": if event.created_at != last_updated: db.set(key, event.created_at) new_items.append(["numix-icon-theme-square"]) new_items.append(["numix-icon-theme-square-kde"]) break return new_items
def check_for_new_items(): db.set('FEED_CHECKED', 'True') db.expire('FEED_CHECKED', 900) new_items = [] gh = login(token=GITHUB_TOKEN) last_id = db.get('ANTBS_GITHUB_LAST_EVENT') or '' repo = gh.repository('numixproject', "numix-icon-theme") commits = repo.commits() latest = None try: commit = commits.next() latest = commit.sha except StopIteration: pass if latest != last_id: db.set('ANTBS_GITHUB_LAST_EVENT', latest) new_items.append(['numix-icon-theme']) gl = Gitlab('https://gitlab.com', GITLAB_TOKEN) gl.auth() nxsq = gl.Project(id='61284') last_updated = db.get('ANTBS_GITLAB_LAST_UPDATED') events = nxsq.Event() for event in events: if event.action_name == 'pushed to': if event.created_at != last_updated: db.set('ANTBS_GITLAB_LAST_UPDATED', event.created_at) new_items.append(['numix-icon-theme-square']) new_items.append(['numix-icon-theme-square-kde']) break if len(new_items) > 0: add_to_build_queue(new_items)
import common import config from github import Github from gitlab import Gitlab import json import os import re import shutil import urllib.request import uuid import zipfile gh = Github(config.github_username, config.github_password) gl = Gitlab('https://gitlab.com', private_token=config.gitlab_private_access_token) gl.auth() def get_latest_release(module): if common.GitService(module['git']['service']) == common.GitService.GitHub: try: repo = gh.get_repo( f'{module["git"]["org_name"]}/{module["git"]["repo_name"]}') except: print( f'[Error] Unable to find repo: {module["git"]["org_name"]}/{module["git"]["repo_name"]}' ) return None releases = repo.get_releases() if releases.totalCount == 0:
__author__ = 'ricard' from redmine import Redmine from gitlab import Gitlab from configparser import ConfigParser config = ConfigParser() config.read_file(open('defaults.cfg')) redmine = Redmine(config.get('redmine', 'url'), key=config.get('redmine', 'key') ) red_project = redmine.project.get(config.get('redmine', 'project')) gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key')) gl.auth() look_for = config.get('gitlab', 'project') for p in gl.Project(per_page=1000): # print(p.path_with_namespace) if p.path_with_namespace == look_for: gl_project_id = p.id gl_project = p break print(gl_project.id) closed_status = [] for status in redmine.issue_status.all(): # print(status, list(status)) if getattr(status, 'is_closed', False): closed_status.append(status.id) trackers = {
def do_poll(self): """Poll Jenkins Server to get Project Information""" while True: # Get a list of ALL Jobs on Jenkins for job_name in self.jenk_client.keys(): jobDetail = self.jenk_client[job_name] lastBuild = jobDetail.get_last_build() buildInfo = BuildItem() if lastBuild.is_running(): buildInfo["status"] = "STARTED" else: buildInfo["status"] = lastBuild.get_status() buildInfo["timestamp"] = lastBuild.get_timestamp().isoformat() buildInfo["label"] = lastBuild.name buildInfo["cause"] = "" # Need a clean way to retrieve this value # TODO: swap GitLab specific library for Git Library gl_client = Gitlab(self.options["gitlab"], self.options["gitlab_key"]) gl_client.auth() ### complete hack.... try: proj_url = jobDetail.get_scm_url()[0] except Exception: continue sep = proj_url.rfind("/") start = proj_url.rfind("/", 0, sep-1) proj_name = proj_url[start+1:] project = None for proj in gl_client.Project(): if proj.path_with_namespace == proj_name: project = proj if project is None: continue # TODO raise error ### TODO remove the above ... soon commit = project.Commit(lastBuild.get_revision()) repoInfo = RepoItem() repoInfo["branch"] = jobDetail.get_scm_branch()[0] repoInfo["commit"] = commit.id repoInfo["blame"] = commit.author_name repoInfo["timestamp"] = commit.created_at ### TODO Refactor to "JobsList" from ProjectsList pushJob = None try: ## We use this order, since it will be in an infinite ## loop, the likely scenario is that we will just be ## updating the status of jobs pushJob = PROJECTS.update(job_name, buildInfo, repoInfo) except IndexError: pushJob = PROJECTS.add(job_name, buildInfo, repoInfo) if pushJob is not None: WEBSOCK_DO_PUSH_EVENT.set(push=pushJob)
def get_gitbranch(auth_user): blists = [] parser = CappConfig(CONF) workdirs = parser.get('PATH', 'workdirs') verconf = CappConfig(os.path.join(workdirs, 'vercnf')) try: gl = Gitlab('http://%s' % parser.get('GITS', 'address'), '%s' % parser.get('GITS', 'addretk')) gl.auth() except: return blists for project in gl.projects.list(page=0, per_page=1000): bdict = {} blist = [] mlist = [] for branch in gl.project_branches.list(project_id=project.id): if not re.match( parser.get('GITS', 'brannot').replace(',', '|'), branch.name): blist.append(branch.name) bdict['bname'] = blist bdict['pname'] = project.name bdict['pnumb'] = '' repo_url[project.name] = project.ssh_url_to_repo #bdict['sshurl']=project.ssh_url_to_repo flag = False for key, val in get_srvlist().viewitems(): if project.name in val.split('-', 1): auth_perm = parser.get('REPO', '%s.permissi' % val).replace('|c', '') flag = True if len(val.split('-', 1)) == 2: if auth_user in auth_perm.split(','): try: mvers = verconf.get('REPO', '%s.versions' % val) except: mvers = '' mlist.append({ 'mnumb': key, 'mname': val.split('-')[0], 'mvers': mvers }) #print mlist else: if auth_user in auth_perm.split(','): bdict['pnumb'] = key else: bdict['pnumb'] = '' if mlist: bdict['mlist'] = mlist else: bdict['mlist'] = '' if not bdict['pnumb'] and not bdict['mlist']: pass else: blists.append(bdict) return blists
def connect_to_gitlab(): gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key')) gl.auth() return gl
class GitlabMonitor(PackageSourceMonitor): def __init__(self, url, token, project=None, mon_type='release', status=None): super().__init__(status) self.type = mon_type self.project_name = project self.last_etag = None self.etag = None self.repo = None self.changed = False if project: self.set_project(project=project, url=url, token=token) def _get_latest(self, what_to_get, pkg_obj=None): if self.project is None: self._project_not_set_error() if 'releases' == what_to_get: what_to_get = 'tags' git_item = getattr(self.project, what_to_get) res = iter(git_item.list()) items_checked = 0 pattern = pkg_obj.mon_match_pattern or '.' # self.logger.debug([git_item, res, pattern]) def _get_next_item(): _latest = etag = '' try: item = next(res).attributes if 'commits' == what_to_get: _latest = item['sha'] etag = item['created_at'] elif 'releases' == what_to_get: _latest = item['release']['tag_name'] etag = item.commit['created_at'] elif 'tags' == what_to_get: _latest = item['release']['tag_name'] etag = item['commit']['created_at'] except StopIteration: pass except Exception as err: self.logger.exception(err) return _latest, etag latest, etag = _get_next_item() if not latest or (pattern and not self._matches_pattern(pattern, latest)): while not latest or (pattern and not self._matches_pattern(pattern, latest)): latest, etag = _get_next_item() items_checked += 1 if items_checked > 50: break if latest and pattern and pattern.startswith('/') and pattern.endswith( '/'): pattern = pattern[1:-1] matches = re.search(pattern, latest) try: latest = matches.group(1) except IndexError: latest = matches.group(0) except Exception: pass if 'commits' != what_to_get and latest.startswith('v'): latest = latest[1:] pkg_obj.mon_etag = etag return latest @staticmethod def _project_not_set_error(): raise AttributeError('project is not set!') def get_latest_commit(self, pattern=None): return self._get_latest('commits', pattern) def get_latest_release(self, pattern=None): return self._get_latest('tags', pattern) def get_latest_tag(self, pattern=None): return self._get_latest('tags', pattern) def package_source_changed(self, pkg_obj, change_type=None, change_id=None): change_type = change_type or pkg_obj.mon_type self.latest = change_id or self._get_latest(change_type, pkg_obj) return super().package_source_changed(pkg_obj) def set_project(self, project=None, url=None, token=None): self.project_name = project if project is not None else self.project_name self.url = url if url is not None else self.url self.token = token if token is not None else self.token if not (self.project_name and self.url and self.token): raise ValueError( 'project, url, and token are required in order to set project!' ) self.gl = Gitlab(url, token) self.gl.auth() self.project = self.gl.projects.get(self.project_name)
class GitlabHelper(object): ACCESS_LEVEL_REFERENCE = { 10: "guest", 20: "reporter", 30: "developer", 40: "maintainer", 50: "maintainer", # NOTE: owner is only usable when your permissions are based on group. } def __init__(self, url, token, timeout, groups, namespace_granularity, admins_group): self.client = None self.gitlab_users = [] self.groups = groups self.timeout = timeout self.token = token self.url = url self.namespace_granularity = namespace_granularity self.admins_group = admins_group self.namespaces = [] def connect(self): """Performs an authentication via private token. Raises: exception: If any errors occurs. """ try: self.client = Gitlab( url=self.url, private_token=self.token, timeout=self.timeout ) self.client.auth() except Exception as e: raise Exception("unable to connect on gitlab :: {}".format(e)) try: if self.namespace_granularity == "group": self.namespaces = self.get_groups() else: self.namespaces = self.get_projects() except Exception as e: raise Exception("unable to define namespaces :: {}".format(e)) def get_projects(self): """Get all projects under the configured namespace (GITLAB_GROUP_SEARCH). Returns: list[gitlab.Project]: list for success, empty otherwise. """ try: projects = [] for group in self.get_groups(): for project in group.projects.list(all=True): projects.append(self.client.projects.get(project.id)) logging.info( "|_ search group={} project={}".format( group.name, project.name ) ) return projects except Exception as e: logging.error("unable to get projects :: {}".format(e)) return [] def get_admins(self): """Returns all admins. e.g. user { 'email': '*****@*****.**', 'id': '123', } Returns: list[dict]: list for success, empty otherwise. """ try: if self.admins_group: ns = self.client.groups.list(search=self.admins_group) return self.get_users(from_namespaces=ns) or [] admins = [] for user in self.client.users.list(all=True): if user.is_admin: admins.append( {"email": user.email, "id": "{}".format(user.id)} ) logging.info( u"|user={} email={} access_level=admin".format( user.name, user.email ) ) return admins except Exception as e: logging.error("unable to retrieve admins :: {}".format(e)) exit(1) return [] def get_users(self, from_namespaces=None): """Returns all users from groups/projects. Args: from_namespaces (list): Retrieve users from this namespaces. e.g. user { 'access_level': 'reporter', 'email': '*****@*****.**', 'id': '123', 'namespace': 'default' } Returns: list[dict]: list for success, empty otherwise. """ try: users = [] namespaces = from_namespaces or self.namespaces for namespace in namespaces: for member in namespace.members.list(all=True): user = self.client.users.get(member.id) users.append( { "access_level": member.access_level, "email": user.email, "id": "{}".format(user.id), "namespace": slugify(namespace.name), } ) logging.info( u"|namespace={} user={} email={} access_level={}".format( namespace.name, user.name, user.email, member.access_level, ) ) return users except Exception as e: logging.error("unable to retrieve users :: {}".format(e)) exit(1) return [] def get_groups(self): groups = [] for group in self.groups: for result in self.client.groups.list(search=group, all=True): if result.parent_id is None: logging.info(u"|found group={}".format(result.name)) groups.append(result) return groups
class GitlabCrawler: def __init__(self, url: str, token: str, user: str, password: str): assert url is None or isinstance(url, str) assert user is None or isinstance(user, str) assert password is None or isinstance(password, str) assert token is None or isinstance(token, str) url = url or 'https://gitlab.com/' if token: self.client = Gitlab(url, private_token=token) else: self.client = Gitlab(url, email=user, password=password) if user and password: self.client.auth() def find(self, query: str, limit: int = None, since: datetime = None, previous: nx.Graph = None): assert query is None or isinstance(query, str) assert limit is None or isinstance(limit, int) and limit >= 0 assert since is None or isinstance(since, datetime) assert previous is None or isinstance(previous, nx.Graph) g = previous or nx.Graph() since = since.isoformat() if since else None graph_lock = Lock() completed = False query_params = dict() if since: query_params['from'] = repr(since) if query: query_params['search'] = query repos = self.client.projects page = 1 page_repos = [] count = 0 while not completed: obey_rate_limit = True def link_user(repo_id: str, user_id: str, **attr): if user_id is None: return with graph_lock: if user_id not in g: g.add_node(user_id, bipartite=1) if (user_id, repo_id) not in g.edges: g.add_edge(user_id, repo_id, **attr) def import_repo(repo): repo_id = repo.path_with_namespace if repo_id is None: return repo, [] with graph_lock: if repo_id in g: return repo, [] if 'forked_from_project' in repo.attributes: d = repo.attributes['forked_from_project'] if d: parent_id = d['path_with_namespace'] parent = repos.get(parent_id) import_repo(parent) link_user(parent_id, repo.namespace['name'], relation='fork', fork_source=repo_id, date=repo.created_at) languages = repo.languages() if hasattr(repo, 'languages') else [] language = sorted(languages.items(), key=lambda t: t[1], reverse=True)[0][0] \ if len(languages) > 0 else '?' weight = repo.star_count or 0 with graph_lock: g.add_node(repo_id, bipartite=0, language=language, weight=weight, date=repo.last_activity_at) repo_forks = [] try: if since is None: if repo.namespace is not None: link_user(repo_id, repo.namespace['name'], relation='owner', date=repo.created_at) contributors = repo.repository_contributors( all=True, obey_rate_limit=False) for user in contributors: user_id = user.get('name') if not user_id or user_id.lower() == 'unknown': user_id = user['email'] link_user(repo_id, user_id, relation='contributor') else: commits = repo.commits.list(all=True, since=since, obey_rate_limit=False) commits = sorted( commits, key=lambda x: dateutil.parser.parse(x.created_at)) for commit in commits: user_id = commit.author_name if not user_id or user_id.lower() == 'unknown': user_id = commit.author_email date = commit.created_at link_user(repo_id, user_id, relation="committer", date=date) repo_forks = [ self.client.projects.get(fork.id) for fork in repo.forks.list(all=True, obey_rate_limit=False) ] except GitlabError: with graph_lock: g.remove_node(repo_id) error: GitlabError _, error, _ = sys.exc_info() if error.response_code == 429: raise except Exception: raise return repo, repo_forks try: print('Finding more repositories.') with concurrent.futures.ThreadPoolExecutor( max_workers=20) as executor: while not limit or count < limit: if not page_repos: page_repos = repos.list( page=page, per_page=30, obey_rate_limit=obey_rate_limit, query_parameters=query_params) obey_rate_limit = False page += 1 if len(page_repos) == 0: break workers = (executor.submit(import_repo, worker) for worker in page_repos) for worker in concurrent.futures.as_completed(workers): repo, repo_forks = worker.result() if repo is not None: print('Analyzed repo {0}.'.format( repo.path_with_namespace or '?')) page_repos.remove(repo) count += 1 page_repos.extend(repo_forks) completed = True except HTTPError: completed = True print(sys.exc_info()) print( 'Communication error with GitHub. Graph completed prematurely.' ) except GitlabGetError: error: GitlabGetError _, error, _ = sys.exc_info() print(error) if error.response_code == 429: print( 'The GitLab rate limit was triggered. Please try again later. ' ) yield g
async def whoami(self, evt: MessageEvent, gl: Gl) -> None: gl.auth() await evt.reply(f"You're logged into {URL(gl.url).host} as " f"[{gl.user.name}]({gl.url}/{gl.user.username})")
def __init__(self, url, project, private_token=None): gl = Gitlab(url, private_token=private_token) gl.auth() self.project = gl.projects.get(gl.search('projects', project)[0]['id'])
def get_projects(config): url = config.get('gitlab', 'url') gl = Gitlab(url, get_token(config)) logger.debug("Connecting to Gitlab {}".format(url)) gl.auth() # XXX catch exceptions user = gl.user name = user.name.encode('utf8') username = user.username.encode('utf8') logger.info("Connected as {1} ({0})".format( user.name.encode('utf8'), user.username)) group = gl.Group(config.get('gitlab', 'group')) # XXX catch exceptions return group.projects