def get_merge_request_overview() -> PullRequestsOverview:
_prs_to_review: List[PullRequest] = []
_prs_authored_with_work: List[PullRequest] = []
_exception = None
_gl = Gitlab(url=GitlabMrsConfig.GITLAB_HOST,
private_token=GitlabMrsConfig.PRIVATE_TOKEN)
_gl.auth()
_author_id = _gl.user.id
mrs = group_mrs(_gl)
try:
_prs_to_review: List[PullRequest] = extract_pull_request_data(
get_merge_requests_to_review(_author_id, mrs))
_prs_authored_with_work: List[PullRequest] = extract_pull_request_data(
get_authored_merge_requests(_author_id, mrs))
except Timeout as e:
_exception = PullRequestException(GitlabMrsConstants.MODULE,
GitlabMrsConstants.TIMEOUT_MESSAGE,
e, traceback.format_exc())
except GitlabHttpError as e:
_exception = PullRequestException(
GitlabMrsConstants.MODULE, GitlabMrsConstants.CONNECTION_MESSAGE,
e, traceback.format_exc())
except Exception as e:
_exception = PullRequestException(GitlabMrsConstants.MODULE,
GitlabMrsConstants.UNKNOWN_MESSAGE,
e, traceback.format_exc())
return PullRequestsOverview.create(_prs_to_review, _prs_authored_with_work,
_exception)
def connect_api(self, max_tries: int = 3):
token = self.token
# user gets max_tries tries to get the token correct
tries = 0
success = False
gl = None
while not success and tries < max_tries:
try:
if not (isinstance(token, str) and len(token) > 0):
raise Exception('Invalid token')
gl = Gitlab(url=self.remote_host,
private_token=token) # type: Gitlab
gl.auth()
success = True
except Exception:
tries += 1
token = self.set_token()
if not success:
print('Could not validate ' + self.TOKEN_NAME + '.')
quit()
self.api = gl
def process_step_4(message):
cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))})
cur = []
cursor4 = dict(cursor3)
for j in cursor4["token"]:
cur.append(decoder(j))
if message.text in cur:
try:
gl = Gitlab('https://git.iu7.bmstu.ru/',
private_token=message.text)
gl.auth()
username = gl.user.username
db.token.find_one_and_update(
{
"id": encoder(str(message.chat.id)),
"token": encoder(cur)
}, {'$set': {
"idGitLab": encoder(username)
}})
except gitlab.GitlabAuthenticationError:
st = open('./static/access_denied.webp', 'rb')
bot.send_sticker(message.chat.id, st)
bot.send_message(
message.chat.id,
"Произошла ошибка при авторизации в GitLab. Проверьте правильность токена",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
else:
bot.send_message(message.chat.id,
"Такого TOKEN нет в твоем списке...",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
def main(args):
repoName = str(args.repopath.stem)
repoOwner = str(Path(str(args.repopath.parent).replace(':', '/')).stem)
config = loadConfig(args.config)
assert config, "could not load configuation at {}".format(str(args.config))
# Gitlab or custom hostname
gl = Gitlab("https://{}".format(config['hostname']),
private_token=args.token)
gl.auth()
ownerId, isUser = getGitlabOwnerId(gl, args.repopath)
if not ownerId:
print("could not find project owner", repoOwner)
if isUser:
if ownerId and repoName not in getGitlabUserRepoNames(gl, ownerId):
createGitlabUserRepo(gl, config, ownerId, repoName, args.repodesc)
else:
print(args.repopath, "already exists")
else:
if ownerId and repoName not in getGitlabUserGroupNames(gl, ownerId):
createGitlabGroupRepo(gl, config, ownerId, repoName, args.repodesc)
else:
print(args.repopath, "already exists")
def clone(self, repo_name, repo_dst: Path):
gl = Gitlab(url=self.url, private_token=self.auth_token)
gl.auth()
current_user = gl.user
username = current_user.username
https_link = f'https://{username}:{self.auth_token}@' + self.url[
8:] + f"/{username}/{repo_name}.git"
return git.Repo.clone_from(https_link, str(repo_dst))
def main():
pipeline_id = int(os.environ["CI_PIPELINE_ID"])
project_id = os.environ["CI_PROJECT_ID"]
branch = os.environ["CI_COMMIT_REF_NAME"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme,
_parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
interesting_status = ["running", "pending"]
futures = []
with ThreadPoolExecutor(max_workers=2) as executor:
for status in interesting_status:
futures.append(
executor.submit(project.pipelines.list,
order_by="id",
sort="desc",
per_page=1,
ref=branch,
status=status))
latest_pipelines = []
for future in futures:
latest_pipelines += future.result()
latest_pipelines = sorted(latest_pipelines,
key=lambda key: key.attributes["id"],
reverse=True)
if len(latest_pipelines) == 0:
print(
"No running pipelines (has a single job been retried?) - continuing"
)
exit(0)
else:
latest_pipeline = latest_pipelines[0]
assert pipeline_id <= latest_pipeline.id
if pipeline_id < latest_pipeline.id:
print(
"Running pipeline (%s) is not the latest (%s) - cancelling self" %
(pipeline_id, latest_pipeline.id),
flush=True)
pipeline = project.pipelines.get(pipeline_id, lazy=True)
pipeline.cancel()
# Call does not block until actioned so going into a sleep from which it won't wake up...
sleep(999)
assert False
else:
print("Running pipeline (%s) is the latest - continuing" %
(pipeline_id, ))
exit(0)
def gitlab_userdata(tokens):
gh = Gitlab("https://gitlab.com", oauth_token=tokens["access_token"])
gh.auth()
gluser = gh.user
return {
"id": gluser.id,
"login": gluser.username,
"name": gluser.name,
"avatar_url": gluser.avatar_url,
}
def get_clone_commands(token, repo_root):
con = Gitlab("http://gitlab.your.domain", token)
con.auth()
for project in con.Project(per_page=200):
url = project.ssh_url_to_repo
subdir = url.split(":")[1].split("/")[0]
cmd = "bash -c '(mkdir -p {repo_root}/{subdir} && cd {repo_root}/{subdir} && git clone {url})'".format(
**locals())
yield cmd
def is_ci_job_running(job_id: int):
project_id = os.environ["CI_PROJECT_ID"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
job = project.jobs.get(job_id)
return job.attributes["status"] == "running"
def is_ci_job_running(job_id: int):
project_id = os.environ["CI_PROJECT_ID"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme,
_parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
job = project.jobs.get(job_id)
return job.attributes["status"] == "running"
def gitlab_connect(token=None):
"""Authenticate the GL wrapper with Gitlab.
Args:
token (str): The Gitlab token to authenticate with.
Defaults to: None.
"""
token = settings.GITLAB_API_TOKEN
try:
gitlab_client = Gitlab("https://gitlab.com", private_token=token)
gitlab_client.auth()
except GitlabHttpError as e:
logger.exception(e)
return gitlab_client
def get_gl_repos(owner, token):
gl = Gitlab(url='https://gitlab.com', private_token=token)
gl.auth()
users = gl.users.list(username=owner)
if len(users) > 0:
user = users[0]
else:
user = gl.groups.get(owner)
repos = user.projects.list(visibility='public')
git_urls = list()
gl_urls = list()
for repo in repos:
git_urls.append(repo.http_url_to_repo)
gl_urls.append(repo.web_url)
return gl_urls, git_urls
async def server_login(self, evt: MessageEvent, url: str,
token: str) -> None:
gl = Gl(url, private_token=token)
try:
gl.auth()
except GitlabAuthenticationError:
await evt.reply("Invalid access token")
return
except Exception as e:
self.log.warning(
f"Unexpected error logging into GitLab {url} for {evt.sender}",
exc_info=True)
await evt.reply(f"GitLab login failed: {e}")
return
self.db.add_login(evt.sender, url, token)
await evt.reply(
f"Successfully logged into GitLab at {url} as {gl.user.name}")
def process_step_2(message):
cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))})
cur = []
cursor4 = dict(cursor3)
for j in cursor4["token"]:
cur.append(decoder(j))
if message.text in cur:
bot.send_message(message.chat.id,
"Данный TOKEN уже есть в нашей базе данных",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
else:
cur.append(message.text)
db.token.find_one_and_update({"id": encoder(str(message.chat.id))},
{'$set': {
"token": encoder(cur)
}})
bot.send_message(message.chat.id,
"Ваш TOKEN был успешно добавлен в нашу базу данных 🎉",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
try:
gl = Gitlab('https://git.iu7.bmstu.ru/',
private_token=message.text)
gl.auth()
username = gl.user.username
db.token.find_one_and_update(
{
"id": encoder(str(message.chat.id)),
"token": encoder(cur)
}, {'$set': {
"idGitLab": encoder(username)
}})
except gitlab.GitlabAuthenticationError:
st4 = open('./static/access_denied.webp', 'rb')
bot.send_sticker(message.chat.id, st4)
bot.send_message(
message.chat.id,
"Произошла ошибка при авторизации в GitLab. Проверьте правильность токена",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
def setUpClass(cls):
cls._gitlab_controller = GitLab8_16_6_ce_0ServiceController()
cls._gitlab_service = cls._gitlab_controller.start_service()
cls.gitlab_location = f"http://{cls._gitlab_service.host}:{cls._gitlab_service.ports[_GITLAB_PORT]}"
gitlab = Gitlab(url=cls.gitlab_location,
email=cls._gitlab_service.root_user.username,
password=cls._gitlab_service.root_user.password)
gitlab.auth()
cls.gitlab_token = gitlab.private_token
cls.project_name = f"{cls._gitlab_service.root_user.username}/{EXAMPLE_PROJECT}"
gitlab.projects.create({"name": EXAMPLE_PROJECT})
project_variables_manager = ProjectVariablesManager(
GitLabConfig(cls.gitlab_location, cls.gitlab_token),
cls.project_name)
project_variables_manager.set(EXAMPLE_VARIABLES)
def crear_repo(package_name):
md5hash_pn = md5(package_name).hexdigest()
first_pref = md5hash_pn[0:2]
second_pref = md5hash_pn[2:4]
workingdir = root_git_dir+"/"+first_pref+"/"+second_pref+"/" + package_name
if not(os.access(root_git_dir+"/"+first_pref, os.F_OK)):
os.mkdir(root_git_dir+"/"+first_pref)
os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
elif not(os.access(root_git_dir+"/"+first_pref+"/"+second_pref, os.F_OK)):
os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
repo = pygit2.init_repository(workingdir)
dashed_package_name=package_name.replace('.','-').lower()
myRemote = repo.remotes.create(package_name, gitlab_url+'/marvin/'+dashed_package_name+'.git')
gl = Gitlab (gitlab_url, gitlab_token)
gl.auth()
p = gl.Project({'name': package_name, 'public':True})
p.save()
return repo
def check_gitlab_repo_for_changes(pkg_obj, build_pkgs):
gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
gl.auth()
project_id = pkg_obj.mon_project
repo = pkg_obj.mon_repo
project = gl.projects.get(project_id)
last_result = pkg_obj.mon_last_result
events = project.events.list()
for event in events:
if event.action_name == 'pushed to':
if event.created_at != last_result:
pkg_obj.mon_last_result = event.created_at
build_pkgs.append('numix-icon-theme-square')
break
return build_pkgs
def GitLab_Work_Log(url, token):
gl = Gitlab(url, token)
gl.auth()
work_log = ''
for project in gl.Project():
work_log += "Project Name: " + project.name + '\n'
for commit in project.Commit():
if commit.author_email == gl.user.email:
commited_at = datetime.strptime(commit.created_at[:19], '%Y-%m-%dT%H:%M:%S')
if datetime.today().date()-timedelta(1) == commited_at.date():
work_log += commit.message + '\n'
for merge in project.MergeRequest():
if merge.author.id == gl.user.id:
merged_at = datetime.strptime(merge.created_at, '%Y-%m-%dT%H:%M:%S.%fZ')
if datetime.today().date()-timedelta(1) == merged_at.date():
work_log += merge.title + '\n'
return work_log
class GitLabModule(NotifyModule):
"""A module to follow gitlab's projects"""
_config = (("url", str, ""), ("token", str, ""),
("ssl_verify", bool, True), ("max_commits", int, 10))
def __init__(self, bot):
NotifyModule.__init__(self,
bot,
name="gitlab",
desc="Gitlab Interface",
delay=60)
self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
self.gl.auth()
# DEBUG
p = GitLabProject(id=31)
pr = self.gl.projects.get(31)
p.update_from_gitlab(pr)
self.bot.session.add(p)
self.bot.session.commit()
def do_action(self):
issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()}
for prj in self.bot.session.query(GitLabProject).all():
gl_prj = self.gl.projects.get(prj.id)
commits = gl_prj.commits.list(per_page=self.max_commits)
last = commits[0].id
while commits and commits[0].id != prj.last_commit:
commit = commits.pop(0)
self.bot.say(MSG['commit'] %
(prj.name, commit.author_name, commit.title))
prj.last_commit = last
for issue in gl_prj.issues.list(all=True):
if issue.id not in issues.keys():
issues[issue.id] = GitLabIssue(id=issue.id)
self.bot.session.add(issues[issue.id])
if issue.state == 'opened':
self.bot.say(
MSG['issue'] %
(prj.name, issue.title, prj.web_url, issue.iid))
issues[issue.id].update_from_gitlab(issue)
self.bot.session.commit()
class GitLabData:
"""GitLab Data object."""
def __init__(self, gitlab_id, priv_token, interval, url):
"""Fetch data from GitLab API for most recent CI job."""
self._gitlab_id = gitlab_id
self._gitlab = Gitlab(url, private_token=priv_token, per_page=1)
self._gitlab.auth()
self.update = Throttle(interval)(self._update)
self.available = False
self.status = None
self.started_at = None
self.finished_at = None
self.duration = None
self.commit_id = None
self.commit_date = None
self.build_id = None
self.branch = None
def _update(self):
try:
_projects = self._gitlab.projects.get(self._gitlab_id)
_last_pipeline = _projects.pipelines.list(page=1)[0]
_last_job = _last_pipeline.jobs.list(page=1)[0]
self.status = _last_pipeline.attributes.get("status")
self.started_at = _last_job.attributes.get("started_at")
self.finished_at = _last_job.attributes.get("finished_at")
self.duration = _last_job.attributes.get("duration")
_commit = _last_job.attributes.get("commit")
self.commit_id = _commit.get("id")
self.commit_date = _commit.get("committed_date")
self.build_id = _last_job.attributes.get("id")
self.branch = _last_job.attributes.get("ref")
self.available = True
except GitlabAuthenticationError as erra:
_LOGGER.error("Authentication Error: %s", erra)
self.available = False
except GitlabGetError as errg:
_LOGGER.error("Project Not Found: %s", errg)
self.available = False
def crear_repo(package_name):
md5hash_pn = md5(package_name).hexdigest()
first_pref = md5hash_pn[0:2]
second_pref = md5hash_pn[2:4]
workingdir = root_git_dir + "/" + first_pref + "/" + second_pref + "/" + package_name
if not (os.access(root_git_dir + "/" + first_pref, os.F_OK)):
os.mkdir(root_git_dir + "/" + first_pref)
os.mkdir(root_git_dir + "/" + first_pref + "/" + second_pref)
elif not (os.access(root_git_dir + "/" + first_pref + "/" + second_pref,
os.F_OK)):
os.mkdir(root_git_dir + "/" + first_pref + "/" + second_pref)
repo = pygit2.init_repository(workingdir)
dashed_package_name = package_name.replace('.', '-').lower()
myRemote = repo.remotes.create(
package_name, gitlab_url + '/marvin/' + dashed_package_name + '.git')
gl = Gitlab(gitlab_url, gitlab_token)
gl.auth()
p = gl.Project({'name': package_name, 'public': True})
p.save()
return repo
def create_gitlab_project(
default_has_issues, default_has_wiki, gitlab_secure_config,
options, project, description, homepage):
created = False
has_issues = 'has-issues' in options or default_has_issues
has_wiki = 'has-wiki' in options or default_has_wiki
secure_config = ConfigParser.ConfigParser()
secure_config.read(gitlab_secure_config)
# Project creation doesn't work via oauth
glab = Gitlab(secure_config.get("gitlab", "url"),
secure_config.get("gitlab", "key"))
glab.auth()
orgs = glab.Group()
orgs_dict = dict(zip([o.name.lower() for o in orgs], orgs))
# Find the project's repo
project_split = project.split('/', 1)
org_name = project_split[0]
if len(project_split) > 1:
repo_name = project_split[1]
else:
repo_name = project
org_name = 'ustack'
try:
org = orgs_dict[org_name.lower()]
except Exception:
# we do not have control of this github org ignore the project.
return False
if glab.search_projects(repo_name):
return created
project_info = {'name': repo_name, 'namespace_id': org.id,
'wiki_enabled': has_wiki, 'description': description,
'issues_enabled': has_issues}
glab.Project(project_info).save()
created = True
return created
class GitlabProject:
def __init__(self, gitlab_url, api_token, namespace, project_slug):
self.gitlab_url = gitlab_url
self.api_token = api_token
self.namespace = namespace
self.project_slug = project_slug
self.client = Gitlab(self.gitlab_url, self.api_token, ssl_verify=False)
@property
def url(self):
return "{}/{}/{}".format(self.gitlab_url, self.namespace,
self.project_slug)
def exists(self):
return len(self.client.projects.list(search=self.project_slug)) > 0
def create(self):
grp = self.client.groups.list(search=self.namespace)[0]
proj_metadata = {'name': self.project_slug, 'namespace_id': grp.id}
proj = self.client.projects.create(proj_metadata,
retry_transient_errors=True)
return proj
def add_issue(self, opts):
project = self.get_project()
user = self.get_current_user()
opts['assignee_id'] = user.id
return project.issues.create(opts)
def get_project(self):
try:
return self.project
except AttributeError:
arg = "{}/{}".format(self.namespace, self.project_slug)
self.project = self.client.projects.get(arg)
return self.project
def get_current_user(self):
self.client.auth()
return self.client.user
def borrar_repo(package_name):
filepath = repo_name(package_name)
gl = Gitlab (gitlab_url, gitlab_token)
gl.auth()
# Project search no anda bien, da error si pongo el package_name entero
# Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar
# asi que buscamos por la ultima palabra del nombre (esperando que no sea Android)
# e iteramos sobre los resultados hasta encontrar package_name
split_name = package_name.split('.')
searchterm = split_name[len(split_name)-1]
projlist = gl.search_projects(searchterm)
for project in projlist:
if project.name == package_name:
break
else:
project = None
if project == None:
raise Exception("El proyecto no existe en GitLab")
else:
project.delete()
rmtree(filepath)
def borrar_repo(package_name):
filepath = repo_name(package_name)
gl = Gitlab(gitlab_url, gitlab_token)
gl.auth()
# Project search no anda bien, da error si pongo el package_name entero
# Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar
# asi que buscamos por la ultima palabra del nombre (esperando que no sea Android)
# e iteramos sobre los resultados hasta encontrar package_name
split_name = package_name.split('.')
searchterm = split_name[len(split_name) - 1]
projlist = gl.search_projects(searchterm)
for project in projlist:
if project.name == package_name:
break
else:
project = None
if project == None:
raise Exception("El proyecto no existe en GitLab")
else:
project.delete()
rmtree(filepath)
class GitLabModule(NotifyModule):
"""A module to follow gitlab's projects"""
_config = (("url", str, ""), ("token", str, ""),
("ssl_verify", bool, True), ("max_commits", int, 10))
def __init__(self, bot):
NotifyModule.__init__(self,
bot,
name="gitlab",
desc="Gitlab Interface",
delay=60)
self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
self.gl.auth()
# DEBUG
p = GitLabProject(id=31)
pr = self.gl.projects.get(31)
p.update_from_gitlab(pr)
self.bot.session.add(p)
self.bot.session.commit()
def do_action(self):
issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()}
for prj in self.bot.session.query(GitLabProject).all():
gl_prj = self.gl.projects.get(prj.id)
commits = gl_prj.commits.list(per_page=self.max_commits)
last = commits[0].id
while commits and commits[0].id != prj.last_commit:
commit = commits.pop(0)
self.bot.say(MSG['commit'] % (prj.name, commit.author_name, commit.title))
prj.last_commit = last
for issue in gl_prj.issues.list(all=True):
if issue.id not in issues.keys():
issues[issue.id] = GitLabIssue(id=issue.id)
self.bot.session.add(issues[issue.id])
if issue.state == 'opened':
self.bot.say(MSG['issue'] % (prj.name, issue.title, prj.web_url, issue.iid))
issues[issue.id].update_from_gitlab(issue)
self.bot.session.commit()
def auth(**kwargs):
'''
Set up gitlab authenticated client
'''
prefix = "gitlab."
# look in kwargs first, then default to config file
def get_key(key, default=None):
return kwargs.get('connection_' + key,
__salt__['config.get'](prefix + key, default))
user = get_key('user', 'admin')
password = get_key('password', 'ADMIN')
token = get_key('token')
url = get_key('url', 'https://localhost/')
LOG.info("Making HTTP request to {0} ...".format(url))
if token:
git = Gitlab(url, token, ssl_verify=False)
else:
git = Gitlab(url, email=user, password=password, ssl_verify=False)
git.auth()
return git
def check_gitlab_repo(project_id=None):
"""
:param project_id:
:return:
"""
new_items = []
gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
gl.auth()
nxsq = gl.projects.get(project_id)
key = 'antbs:monitor:gitlab:{0}'.format(project_id)
last_updated = db.get(key)
events = nxsq.events.list()
for event in events:
if event.action_name == 'pushed to':
if event.created_at != last_updated:
db.set(key, event.created_at)
new_items = ['numix-icon-theme-square']
break
return new_items
def check_gitlab_repo(project_id=None):
"""
:param project_id:
:return:
"""
new_items = []
gl = Gitlab("https://gitlab.com", GITLAB_TOKEN)
gl.auth()
nxsq = gl.Project(id=project_id)
key = "antbs:monitor:gitlab:%s" % project_id
last_updated = db.get(key)
events = nxsq.Event()
for event in events:
if event.action_name == "pushed to":
if event.created_at != last_updated:
db.set(key, event.created_at)
new_items.append(["numix-icon-theme-square"])
new_items.append(["numix-icon-theme-square-kde"])
break
return new_items
def check_for_new_items():
db.set('FEED_CHECKED', 'True')
db.expire('FEED_CHECKED', 900)
new_items = []
gh = login(token=GITHUB_TOKEN)
last_id = db.get('ANTBS_GITHUB_LAST_EVENT') or ''
repo = gh.repository('numixproject', "numix-icon-theme")
commits = repo.commits()
latest = None
try:
commit = commits.next()
latest = commit.sha
except StopIteration:
pass
if latest != last_id:
db.set('ANTBS_GITHUB_LAST_EVENT', latest)
new_items.append(['numix-icon-theme'])
gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
gl.auth()
nxsq = gl.Project(id='61284')
last_updated = db.get('ANTBS_GITLAB_LAST_UPDATED')
events = nxsq.Event()
for event in events:
if event.action_name == 'pushed to':
if event.created_at != last_updated:
db.set('ANTBS_GITLAB_LAST_UPDATED', event.created_at)
new_items.append(['numix-icon-theme-square'])
new_items.append(['numix-icon-theme-square-kde'])
break
if len(new_items) > 0:
add_to_build_queue(new_items)
import common
import config
from github import Github
from gitlab import Gitlab
import json
import os
import re
import shutil
import urllib.request
import uuid
import zipfile
gh = Github(config.github_username, config.github_password)
gl = Gitlab('https://gitlab.com',
private_token=config.gitlab_private_access_token)
gl.auth()
def get_latest_release(module):
if common.GitService(module['git']['service']) == common.GitService.GitHub:
try:
repo = gh.get_repo(
f'{module["git"]["org_name"]}/{module["git"]["repo_name"]}')
except:
print(
f'[Error] Unable to find repo: {module["git"]["org_name"]}/{module["git"]["repo_name"]}'
)
return None
releases = repo.get_releases()
if releases.totalCount == 0:
__author__ = 'ricard'
from redmine import Redmine
from gitlab import Gitlab
from configparser import ConfigParser
config = ConfigParser()
config.read_file(open('defaults.cfg'))
redmine = Redmine(config.get('redmine', 'url'), key=config.get('redmine', 'key') )
red_project = redmine.project.get(config.get('redmine', 'project'))
gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
gl.auth()
look_for = config.get('gitlab', 'project')
for p in gl.Project(per_page=1000):
# print(p.path_with_namespace)
if p.path_with_namespace == look_for:
gl_project_id = p.id
gl_project = p
break
print(gl_project.id)
closed_status = []
for status in redmine.issue_status.all():
# print(status, list(status))
if getattr(status, 'is_closed', False):
closed_status.append(status.id)
trackers = {
def do_poll(self):
"""Poll Jenkins Server to get Project Information"""
while True:
# Get a list of ALL Jobs on Jenkins
for job_name in self.jenk_client.keys():
jobDetail = self.jenk_client[job_name]
lastBuild = jobDetail.get_last_build()
buildInfo = BuildItem()
if lastBuild.is_running():
buildInfo["status"] = "STARTED"
else:
buildInfo["status"] = lastBuild.get_status()
buildInfo["timestamp"] = lastBuild.get_timestamp().isoformat()
buildInfo["label"] = lastBuild.name
buildInfo["cause"] = "" # Need a clean way to retrieve this value
# TODO: swap GitLab specific library for Git Library
gl_client = Gitlab(self.options["gitlab"], self.options["gitlab_key"])
gl_client.auth()
### complete hack....
try:
proj_url = jobDetail.get_scm_url()[0]
except Exception:
continue
sep = proj_url.rfind("/")
start = proj_url.rfind("/", 0, sep-1)
proj_name = proj_url[start+1:]
project = None
for proj in gl_client.Project():
if proj.path_with_namespace == proj_name:
project = proj
if project is None:
continue # TODO raise error
### TODO remove the above ... soon
commit = project.Commit(lastBuild.get_revision())
repoInfo = RepoItem()
repoInfo["branch"] = jobDetail.get_scm_branch()[0]
repoInfo["commit"] = commit.id
repoInfo["blame"] = commit.author_name
repoInfo["timestamp"] = commit.created_at
### TODO Refactor to "JobsList" from ProjectsList
pushJob = None
try:
## We use this order, since it will be in an infinite
## loop, the likely scenario is that we will just be
## updating the status of jobs
pushJob = PROJECTS.update(job_name, buildInfo, repoInfo)
except IndexError:
pushJob = PROJECTS.add(job_name, buildInfo, repoInfo)
if pushJob is not None:
WEBSOCK_DO_PUSH_EVENT.set(push=pushJob)
def get_gitbranch(auth_user):
blists = []
parser = CappConfig(CONF)
workdirs = parser.get('PATH', 'workdirs')
verconf = CappConfig(os.path.join(workdirs, 'vercnf'))
try:
gl = Gitlab('http://%s' % parser.get('GITS', 'address'),
'%s' % parser.get('GITS', 'addretk'))
gl.auth()
except:
return blists
for project in gl.projects.list(page=0, per_page=1000):
bdict = {}
blist = []
mlist = []
for branch in gl.project_branches.list(project_id=project.id):
if not re.match(
parser.get('GITS', 'brannot').replace(',', '|'),
branch.name):
blist.append(branch.name)
bdict['bname'] = blist
bdict['pname'] = project.name
bdict['pnumb'] = ''
repo_url[project.name] = project.ssh_url_to_repo
#bdict['sshurl']=project.ssh_url_to_repo
flag = False
for key, val in get_srvlist().viewitems():
if project.name in val.split('-', 1):
auth_perm = parser.get('REPO',
'%s.permissi' % val).replace('|c', '')
flag = True
if len(val.split('-', 1)) == 2:
if auth_user in auth_perm.split(','):
try:
mvers = verconf.get('REPO', '%s.versions' % val)
except:
mvers = ''
mlist.append({
'mnumb': key,
'mname': val.split('-')[0],
'mvers': mvers
})
#print mlist
else:
if auth_user in auth_perm.split(','):
bdict['pnumb'] = key
else:
bdict['pnumb'] = ''
if mlist: bdict['mlist'] = mlist
else: bdict['mlist'] = ''
if not bdict['pnumb'] and not bdict['mlist']: pass
else: blists.append(bdict)
return blists
def connect_to_gitlab():
gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
gl.auth()
return gl
class GitlabMonitor(PackageSourceMonitor):
def __init__(self,
url,
token,
project=None,
mon_type='release',
status=None):
super().__init__(status)
self.type = mon_type
self.project_name = project
self.last_etag = None
self.etag = None
self.repo = None
self.changed = False
if project:
self.set_project(project=project, url=url, token=token)
def _get_latest(self, what_to_get, pkg_obj=None):
if self.project is None:
self._project_not_set_error()
if 'releases' == what_to_get:
what_to_get = 'tags'
git_item = getattr(self.project, what_to_get)
res = iter(git_item.list())
items_checked = 0
pattern = pkg_obj.mon_match_pattern or '.'
# self.logger.debug([git_item, res, pattern])
def _get_next_item():
_latest = etag = ''
try:
item = next(res).attributes
if 'commits' == what_to_get:
_latest = item['sha']
etag = item['created_at']
elif 'releases' == what_to_get:
_latest = item['release']['tag_name']
etag = item.commit['created_at']
elif 'tags' == what_to_get:
_latest = item['release']['tag_name']
etag = item['commit']['created_at']
except StopIteration:
pass
except Exception as err:
self.logger.exception(err)
return _latest, etag
latest, etag = _get_next_item()
if not latest or (pattern
and not self._matches_pattern(pattern, latest)):
while not latest or (pattern and
not self._matches_pattern(pattern, latest)):
latest, etag = _get_next_item()
items_checked += 1
if items_checked > 50:
break
if latest and pattern and pattern.startswith('/') and pattern.endswith(
'/'):
pattern = pattern[1:-1]
matches = re.search(pattern, latest)
try:
latest = matches.group(1)
except IndexError:
latest = matches.group(0)
except Exception:
pass
if 'commits' != what_to_get and latest.startswith('v'):
latest = latest[1:]
pkg_obj.mon_etag = etag
return latest
@staticmethod
def _project_not_set_error():
raise AttributeError('project is not set!')
def get_latest_commit(self, pattern=None):
return self._get_latest('commits', pattern)
def get_latest_release(self, pattern=None):
return self._get_latest('tags', pattern)
def get_latest_tag(self, pattern=None):
return self._get_latest('tags', pattern)
def package_source_changed(self,
pkg_obj,
change_type=None,
change_id=None):
change_type = change_type or pkg_obj.mon_type
self.latest = change_id or self._get_latest(change_type, pkg_obj)
return super().package_source_changed(pkg_obj)
def set_project(self, project=None, url=None, token=None):
self.project_name = project if project is not None else self.project_name
self.url = url if url is not None else self.url
self.token = token if token is not None else self.token
if not (self.project_name and self.url and self.token):
raise ValueError(
'project, url, and token are required in order to set project!'
)
self.gl = Gitlab(url, token)
self.gl.auth()
self.project = self.gl.projects.get(self.project_name)
class GitlabHelper(object):
ACCESS_LEVEL_REFERENCE = {
10: "guest",
20: "reporter",
30: "developer",
40: "maintainer",
50: "maintainer", # NOTE: owner is only usable when your permissions are based on group.
}
def __init__(self, url, token, timeout, groups, namespace_granularity, admins_group):
self.client = None
self.gitlab_users = []
self.groups = groups
self.timeout = timeout
self.token = token
self.url = url
self.namespace_granularity = namespace_granularity
self.admins_group = admins_group
self.namespaces = []
def connect(self):
"""Performs an authentication via private token.
Raises:
exception: If any errors occurs.
"""
try:
self.client = Gitlab(
url=self.url, private_token=self.token, timeout=self.timeout
)
self.client.auth()
except Exception as e:
raise Exception("unable to connect on gitlab :: {}".format(e))
try:
if self.namespace_granularity == "group":
self.namespaces = self.get_groups()
else:
self.namespaces = self.get_projects()
except Exception as e:
raise Exception("unable to define namespaces :: {}".format(e))
def get_projects(self):
"""Get all projects under the configured namespace (GITLAB_GROUP_SEARCH).
Returns:
list[gitlab.Project]: list for success, empty otherwise.
"""
try:
projects = []
for group in self.get_groups():
for project in group.projects.list(all=True):
projects.append(self.client.projects.get(project.id))
logging.info(
"|_ search group={} project={}".format(
group.name, project.name
)
)
return projects
except Exception as e:
logging.error("unable to get projects :: {}".format(e))
return []
def get_admins(self):
"""Returns all admins.
e.g. user {
'email': '*****@*****.**',
'id': '123',
}
Returns:
list[dict]: list for success, empty otherwise.
"""
try:
if self.admins_group:
ns = self.client.groups.list(search=self.admins_group)
return self.get_users(from_namespaces=ns) or []
admins = []
for user in self.client.users.list(all=True):
if user.is_admin:
admins.append(
{"email": user.email, "id": "{}".format(user.id)}
)
logging.info(
u"|user={} email={} access_level=admin".format(
user.name, user.email
)
)
return admins
except Exception as e:
logging.error("unable to retrieve admins :: {}".format(e))
exit(1)
return []
def get_users(self, from_namespaces=None):
"""Returns all users from groups/projects.
Args:
from_namespaces (list): Retrieve users from this namespaces.
e.g. user {
'access_level': 'reporter',
'email': '*****@*****.**',
'id': '123',
'namespace': 'default'
}
Returns:
list[dict]: list for success, empty otherwise.
"""
try:
users = []
namespaces = from_namespaces or self.namespaces
for namespace in namespaces:
for member in namespace.members.list(all=True):
user = self.client.users.get(member.id)
users.append(
{
"access_level": member.access_level,
"email": user.email,
"id": "{}".format(user.id),
"namespace": slugify(namespace.name),
}
)
logging.info(
u"|namespace={} user={} email={} access_level={}".format(
namespace.name,
user.name,
user.email,
member.access_level,
)
)
return users
except Exception as e:
logging.error("unable to retrieve users :: {}".format(e))
exit(1)
return []
def get_groups(self):
groups = []
for group in self.groups:
for result in self.client.groups.list(search=group, all=True):
if result.parent_id is None:
logging.info(u"|found group={}".format(result.name))
groups.append(result)
return groups
class GitlabCrawler:
def __init__(self, url: str, token: str, user: str, password: str):
assert url is None or isinstance(url, str)
assert user is None or isinstance(user, str)
assert password is None or isinstance(password, str)
assert token is None or isinstance(token, str)
url = url or 'https://gitlab.com/'
if token:
self.client = Gitlab(url, private_token=token)
else:
self.client = Gitlab(url, email=user, password=password)
if user and password:
self.client.auth()
def find(self,
query: str,
limit: int = None,
since: datetime = None,
previous: nx.Graph = None):
assert query is None or isinstance(query, str)
assert limit is None or isinstance(limit, int) and limit >= 0
assert since is None or isinstance(since, datetime)
assert previous is None or isinstance(previous, nx.Graph)
g = previous or nx.Graph()
since = since.isoformat() if since else None
graph_lock = Lock()
completed = False
query_params = dict()
if since:
query_params['from'] = repr(since)
if query:
query_params['search'] = query
repos = self.client.projects
page = 1
page_repos = []
count = 0
while not completed:
obey_rate_limit = True
def link_user(repo_id: str, user_id: str, **attr):
if user_id is None:
return
with graph_lock:
if user_id not in g:
g.add_node(user_id, bipartite=1)
if (user_id, repo_id) not in g.edges:
g.add_edge(user_id, repo_id, **attr)
def import_repo(repo):
repo_id = repo.path_with_namespace
if repo_id is None:
return repo, []
with graph_lock:
if repo_id in g:
return repo, []
if 'forked_from_project' in repo.attributes:
d = repo.attributes['forked_from_project']
if d:
parent_id = d['path_with_namespace']
parent = repos.get(parent_id)
import_repo(parent)
link_user(parent_id,
repo.namespace['name'],
relation='fork',
fork_source=repo_id,
date=repo.created_at)
languages = repo.languages() if hasattr(repo,
'languages') else []
language = sorted(languages.items(), key=lambda t: t[1], reverse=True)[0][0] \
if len(languages) > 0 else '?'
weight = repo.star_count or 0
with graph_lock:
g.add_node(repo_id,
bipartite=0,
language=language,
weight=weight,
date=repo.last_activity_at)
repo_forks = []
try:
if since is None:
if repo.namespace is not None:
link_user(repo_id,
repo.namespace['name'],
relation='owner',
date=repo.created_at)
contributors = repo.repository_contributors(
all=True, obey_rate_limit=False)
for user in contributors:
user_id = user.get('name')
if not user_id or user_id.lower() == 'unknown':
user_id = user['email']
link_user(repo_id, user_id, relation='contributor')
else:
commits = repo.commits.list(all=True,
since=since,
obey_rate_limit=False)
commits = sorted(
commits,
key=lambda x: dateutil.parser.parse(x.created_at))
for commit in commits:
user_id = commit.author_name
if not user_id or user_id.lower() == 'unknown':
user_id = commit.author_email
date = commit.created_at
link_user(repo_id,
user_id,
relation="committer",
date=date)
repo_forks = [
self.client.projects.get(fork.id)
for fork in repo.forks.list(all=True,
obey_rate_limit=False)
]
except GitlabError:
with graph_lock:
g.remove_node(repo_id)
error: GitlabError
_, error, _ = sys.exc_info()
if error.response_code == 429:
raise
except Exception:
raise
return repo, repo_forks
try:
print('Finding more repositories.')
with concurrent.futures.ThreadPoolExecutor(
max_workers=20) as executor:
while not limit or count < limit:
if not page_repos:
page_repos = repos.list(
page=page,
per_page=30,
obey_rate_limit=obey_rate_limit,
query_parameters=query_params)
obey_rate_limit = False
page += 1
if len(page_repos) == 0:
break
workers = (executor.submit(import_repo, worker)
for worker in page_repos)
for worker in concurrent.futures.as_completed(workers):
repo, repo_forks = worker.result()
if repo is not None:
print('Analyzed repo {0}.'.format(
repo.path_with_namespace or '?'))
page_repos.remove(repo)
count += 1
page_repos.extend(repo_forks)
completed = True
except HTTPError:
completed = True
print(sys.exc_info())
print(
'Communication error with GitHub. Graph completed prematurely.'
)
except GitlabGetError:
error: GitlabGetError
_, error, _ = sys.exc_info()
print(error)
if error.response_code == 429:
print(
'The GitLab rate limit was triggered. Please try again later. '
)
yield g
async def whoami(self, evt: MessageEvent, gl: Gl) -> None:
gl.auth()
await evt.reply(f"You're logged into {URL(gl.url).host} as "
f"[{gl.user.name}]({gl.url}/{gl.user.username})")
def __init__(self, url, project, private_token=None):
gl = Gitlab(url, private_token=private_token)
gl.auth()
self.project = gl.projects.get(gl.search('projects', project)[0]['id'])
def get_projects(config):
url = config.get('gitlab', 'url')
gl = Gitlab(url, get_token(config))
logger.debug("Connecting to Gitlab {}".format(url))
gl.auth() # XXX catch exceptions
user = gl.user
name = user.name.encode('utf8')
username = user.username.encode('utf8')
logger.info("Connected as {1} ({0})".format(
user.name.encode('utf8'), user.username))
group = gl.Group(config.get('gitlab', 'group')) # XXX catch exceptions
return group.projects