def login(self, token):
"""Login to Gitlab.
:param token: The user's Gitlab private token.
"""
self._gitlab = Gitlab(self._url, token)
def __init__(self, account):
id = 0
for key in j.application.config.getKeysFromPrefix(
"gitlabclient.server"):
# key=key.replace("gitlabclient.server.","")
if key.find("name") <> -1:
if j.application.config.get(key) == account:
key2 = key.replace("gitlabclient.server.", "")
id = key2.split(".")[0]
if id == 0:
raise RuntimeError("Did not find account:%s for gitlab" % account)
prefix = "gitlabclient.server.%s" % id
self.addr = j.application.config.get("%s.addr" % prefix)
# self.accountPathLocal = j.system.fs.joinPaths("/opt/code",accountName)
# j.system.fs.createDir(self.accountPathLocal)
# self._gitlab = gitlab.Gitlab(self.addr)
login = j.application.config.get("%s.login" % prefix)
passwd = j.application.config.get("%s.passwd" % prefix)
self.passwd = passwd
if passwd <> "":
Gitlab.__init__(self, self.addr) #, token=token)
self.login(login, passwd)
self.load()
# for item in dir(self._gitlab):
# if item[0]<>"_":
# setattr(self,item,getattr(self._gitlab,item))
self.gitclients = {}
self.loginName = login
self.port = 80
def connect_api(self, max_tries: int = 3):
token = self.token
# user gets max_tries tries to get the token correct
tries = 0
success = False
gl = None
while not success and tries < max_tries:
try:
if not (isinstance(token, str) and len(token) > 0):
raise Exception('Invalid token')
gl = Gitlab(url=self.remote_host,
private_token=token) # type: Gitlab
gl.auth()
success = True
except Exception:
tries += 1
token = self.set_token()
if not success:
print('Could not validate ' + self.TOKEN_NAME + '.')
quit()
self.api = gl
def __init__(self, name, token=None, url=None, ssh=None):
super(GitLabRepository,
self).__init__(os.path.join(StateHolder.home_dir, 'gitLab',
name))
target_url = url if url is not None else "http://gitlab.com"
if token is None:
ColorPrint.exit_after_print_messages(
message="Gitlab configuration is empty in section " + name +
"!")
else:
self.gitlab = Gitlab(target_url, private_token=token)
self.gitlab.version() # test connection
lst = dict()
projects = self.gitlab.projects.list(membership=True)
for project in projects:
project_name = project.name
lst[project_name] = dict()
lst[project_name]['git'] = str(project.ssh_url_to_repo)
if ssh is not None:
lst[project_name]['ssh'] = ssh
self.write_yaml_file(os.path.join(self.target_dir,
'pocok-catalog.yml'),
yaml.dump(data=lst, default_flow_style=False),
create=True)
def __init__(self):
FeatureExtractor.__init__(self)
project = Gitlab('https://jugit.fz-juelich.de').projects.get(3009)
jsonfiles = [
f['name'] for f in project.repository_tree()
if f['type'] == 'blob' and f['name'].endswith('json')
]
minval = maxval = 0
new_profiles = []
for jsonfile in jsonfiles:
f = project.files.get(file_path=jsonfile, ref='master')
data = json.loads(f.decode())
src_name = data['name']
src_info = data['description']
src_file = jsonfile
parcellation = parcellations[data['parcellation id']]
regions_available = data['data']['field names']
for region in regions_available:
profile = data['data']['profiles'][region]
if max(profile) > maxval:
maxval = max(profile)
if min(profile) > minval:
minval = min(profile)
new_profiles.append(
ConnectivityProfile(region, profile, regions_available,
src_name, src_info, src_file,
parcellation))
for profile in new_profiles:
profile.globalrange = (minval, maxval)
self.register(profile)
def test_invalid_auth_args():
with pytest.raises(ValueError):
Gitlab(
"http://localhost",
api_version="4",
private_token="private_token",
oauth_token="bearer",
)
with pytest.raises(ValueError):
Gitlab(
"http://localhost",
api_version="4",
oauth_token="bearer",
http_username="******",
http_password="******",
)
with pytest.raises(ValueError):
Gitlab(
"http://localhost",
api_version="4",
private_token="private_token",
http_password="******",
)
with pytest.raises(ValueError):
Gitlab(
"http://localhost",
api_version="4",
private_token="private_token",
http_username="******",
)
def __init__(self, headers):
self.authenticated = all(
[header in headers.keys() for header in self.auth_headers])
if not self.authenticated:
return
parsed_id_token = self.parse_jwt_from_headers(headers)
self.keycloak_user_id = parsed_id_token["sub"]
self.email = parsed_id_token["email"]
self.full_name = parsed_id_token["name"]
self.username = parsed_id_token["preferred_username"]
self.safe_username = escapism.escape(self.username,
escape_char="-").lower()
self.oidc_issuer = parsed_id_token["iss"]
(
self.git_url,
self.git_auth_header,
self.git_token,
) = self.git_creds_from_headers(headers)
self.gitlab_client = Gitlab(
self.git_url,
api_version=4,
oauth_token=self.git_token,
)
self.setup_k8s()
def setUp(self):
self.user = os.environ.get('gitlab_user', 'root')
self.password = os.environ.get('gitlab_password', '1WmA1a3ONs9F')
self.host = os.environ.get('gitlab_host', 'http://localhost:10080')
self.gitlab = Gitlab(host=self.host, verify_ssl=False)
self.gitlab.login(user=self.user, password=self.password)
def get_merge_request_overview() -> PullRequestsOverview:
_prs_to_review: List[PullRequest] = []
_prs_authored_with_work: List[PullRequest] = []
_exception = None
_gl = Gitlab(url=GitlabMrsConfig.GITLAB_HOST,
private_token=GitlabMrsConfig.PRIVATE_TOKEN)
_gl.auth()
_author_id = _gl.user.id
mrs = group_mrs(_gl)
try:
_prs_to_review: List[PullRequest] = extract_pull_request_data(
get_merge_requests_to_review(_author_id, mrs))
_prs_authored_with_work: List[PullRequest] = extract_pull_request_data(
get_authored_merge_requests(_author_id, mrs))
except Timeout as e:
_exception = PullRequestException(GitlabMrsConstants.MODULE,
GitlabMrsConstants.TIMEOUT_MESSAGE,
e, traceback.format_exc())
except GitlabHttpError as e:
_exception = PullRequestException(
GitlabMrsConstants.MODULE, GitlabMrsConstants.CONNECTION_MESSAGE,
e, traceback.format_exc())
except Exception as e:
_exception = PullRequestException(GitlabMrsConstants.MODULE,
GitlabMrsConstants.UNKNOWN_MESSAGE,
e, traceback.format_exc())
return PullRequestsOverview.create(_prs_to_review, _prs_authored_with_work,
_exception)
def process_step_4(message):
cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))})
cur = []
cursor4 = dict(cursor3)
for j in cursor4["token"]:
cur.append(decoder(j))
if message.text in cur:
try:
gl = Gitlab('https://git.iu7.bmstu.ru/',
private_token=message.text)
gl.auth()
username = gl.user.username
db.token.find_one_and_update(
{
"id": encoder(str(message.chat.id)),
"token": encoder(cur)
}, {'$set': {
"idGitLab": encoder(username)
}})
except gitlab.GitlabAuthenticationError:
st = open('./static/access_denied.webp', 'rb')
bot.send_sticker(message.chat.id, st)
bot.send_message(
message.chat.id,
"Произошла ошибка при авторизации в GitLab. Проверьте правильность токена",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
else:
bot.send_message(message.chat.id,
"Такого TOKEN нет в твоем списке...",
parse_mode="html",
reply_markup=types.ReplyKeyboardRemove())
def connect_to_gitlab(gitlab_hostname,
username,
password,
gitlab_ssl=True,
verify_ssl=True):
log("Connecting to GitLab")
if not gitlab_ssl:
hostname = "http://{}".format(gitlab_hostname)
else:
hostname = "https://{}".format(gitlab_hostname)
try:
g = Gitlab(hostname, verify_ssl=verify_ssl)
rv = g.login(username, password)
if not rv:
error(
"Could not connect to Git server ({}). Reason unknown.".format(
gitlab_hostname))
else:
return g
except requests.exceptions.SSLError:
error_msg = "Your computer is not set up to trust the CS department's SSL certificate.\n"
error_msg += "Try running the setup script with the --skip-ssl-verify option."
error(error_msg)
except HttpError as he:
if str(he) == "401 Unauthorized":
error(
"Could not connect to Git server (incorrect username/password)"
)
else:
error(
"Unexpected error while connecting to Git server (Reason: {})".
format(he))
def main(args):
repoName = str(args.repopath.stem)
repoOwner = str(Path(str(args.repopath.parent).replace(':', '/')).stem)
config = loadConfig(args.config)
assert config, "could not load configuation at {}".format(str(args.config))
# Gitlab or custom hostname
gl = Gitlab("https://{}".format(config['hostname']),
private_token=args.token)
gl.auth()
ownerId, isUser = getGitlabOwnerId(gl, args.repopath)
if not ownerId:
print("could not find project owner", repoOwner)
if isUser:
if ownerId and repoName not in getGitlabUserRepoNames(gl, ownerId):
createGitlabUserRepo(gl, config, ownerId, repoName, args.repodesc)
else:
print(args.repopath, "already exists")
else:
if ownerId and repoName not in getGitlabUserGroupNames(gl, ownerId):
createGitlabGroupRepo(gl, config, ownerId, repoName, args.repodesc)
else:
print(args.repopath, "already exists")
def connect(self):
self.gl = Gitlab(self.gitlab['endpoint'],
self.gitlab['token'],
ssl_verify=False)
self.gl.auth()
self.kb = Kanboard(self.kanboard['endpoint'], self.kanboard['user'],
self.kanboard['token'])
def main():
pipeline_id = int(os.environ["CI_PIPELINE_ID"])
project_id = os.environ["CI_PROJECT_ID"]
branch = os.environ["CI_COMMIT_REF_NAME"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme,
_parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
interesting_status = ["running", "pending"]
futures = []
with ThreadPoolExecutor(max_workers=2) as executor:
for status in interesting_status:
futures.append(
executor.submit(project.pipelines.list,
order_by="id",
sort="desc",
per_page=1,
ref=branch,
status=status))
latest_pipelines = []
for future in futures:
latest_pipelines += future.result()
latest_pipelines = sorted(latest_pipelines,
key=lambda key: key.attributes["id"],
reverse=True)
if len(latest_pipelines) == 0:
print(
"No running pipelines (has a single job been retried?) - continuing"
)
exit(0)
else:
latest_pipeline = latest_pipelines[0]
assert pipeline_id <= latest_pipeline.id
if pipeline_id < latest_pipeline.id:
print(
"Running pipeline (%s) is not the latest (%s) - cancelling self" %
(pipeline_id, latest_pipeline.id),
flush=True)
pipeline = project.pipelines.get(pipeline_id, lazy=True)
pipeline.cancel()
# Call does not block until actioned so going into a sleep from which it won't wake up...
sleep(999)
assert False
else:
print("Running pipeline (%s) is the latest - continuing" %
(pipeline_id, ))
exit(0)
def auth(self):
self.conn = Gitlab(
self.url,
private_token=self.private_token,
oauth_token=self.oauth_token,
timeout=self.timeout,
per_page=self.per_page,
)
self.conn.auth()
def clone(self, repo_name, repo_dst: Path):
gl = Gitlab(url=self.url, private_token=self.auth_token)
gl.auth()
current_user = gl.user
username = current_user.username
https_link = f'https://{username}:{self.auth_token}@' + self.url[
8:] + f"/{username}/{repo_name}.git"
return git.Repo.clone_from(https_link, str(repo_dst))
def setUp(self):
self.user = os.environ.get('gitlab_user', 'root')
self.password = os.environ.get('gitlab_password', '5iveL!fe')
self.host = os.environ.get('gitlab_host', 'http://gitlab:80')
self.gitlab = Gitlab(host=self.host,
verify_ssl=False,
suppress_http_error=False)
self.gitlab.login(user=self.user, password=self.password)
def gitlab_userdata(tokens):
gh = Gitlab("https://gitlab.com", oauth_token=tokens["access_token"])
gh.auth()
gluser = gh.user
return {
"id": gluser.id,
"login": gluser.username,
"name": gluser.name,
"avatar_url": gluser.avatar_url,
}
def get_clone_commands(token, repo_root):
con = Gitlab("http://gitlab.your.domain", token)
con.auth()
for project in con.Project(per_page=200):
url = project.ssh_url_to_repo
subdir = url.split(":")[1].split("/")[0]
cmd = "bash -c '(mkdir -p {repo_root}/{subdir} && cd {repo_root}/{subdir} && git clone {url})'".format(
**locals())
yield cmd
def __init__(self, url: str, token: str, user: str, password: str):
assert url is None or isinstance(url, str)
assert user is None or isinstance(user, str)
assert password is None or isinstance(password, str)
assert token is None or isinstance(token, str)
url = url or 'https://gitlab.com/'
if token:
self.client = Gitlab(url, private_token=token)
else:
self.client = Gitlab(url, email=user, password=password)
if user and password:
self.client.auth()
def get_credentials(self, username, password, delete_repo = False):
try:
g = Gitlab(self.gitlab_hostname)
rv = g.login(username, password)
if not rv:
return None, False
else:
return g.headers["PRIVATE-TOKEN"], True
except gitlab.exceptions.HttpError as he:
if str(he) == "401 Unauthorized":
return None, False
else:
raise ChisubmitException("Unexpected error getting authorization token (Reason: %s)" % (he), he)
def is_ci_job_running(job_id: int):
project_id = os.environ["CI_PROJECT_ID"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
job = project.jobs.get(job_id)
return job.attributes["status"] == "running"
def _main(args: argparse.Namespace):
"""Pass arguments to respective function."""
__log__.info('------- Arguments: -------')
__log__.info('OUTDIR: %s', args.OUTDIR)
__log__.info('REPOSITORY_LIST: %s', args.REPOSITORY_LIST.name)
__log__.info('--gitlab-repos-dir: %s', args.gitlab_repos_dir)
__log__.info('--gitlab-host: %s', args.gitlab_host)
__log__.info('------- Arguments end -------')
gitlab = Gitlab(args.gitlab_host, api_version=4)
gitlab.repository_prefix = args.gitlab_repos_dir
store_repository_info(args.REPOSITORY_LIST, gitlab, args.OUTDIR)
def gitlab_connect(token=None):
"""Authenticate the GL wrapper with Gitlab.
Args:
token (str): The Gitlab token to authenticate with.
Defaults to: None.
"""
token = settings.GITLAB_API_TOKEN
try:
gitlab_client = Gitlab("https://gitlab.com", private_token=token)
gitlab_client.auth()
except GitlabHttpError as e:
logger.exception(e)
return gitlab_client
def setUp(self):
self.user = os.environ.get('gitlab_user', 'root')
self.password = os.environ.get('gitlab_password', '1WmA1a3ONs9F')
self.host = os.environ.get('gitlab_host', 'http://localhost:10080')
self.gitlab = Gitlab(host=self.host, verify_ssl=False)
self.gitlab.host = self.host
responses.add(responses.POST,
self.gitlab.api_url + '/session',
json={'private_token': 'test'},
status=201,
content_type='application/json')
self.gitlab.login(user=self.user, password=self.password)
def is_ci_job_running(job_id: int):
project_id = os.environ["CI_PROJECT_ID"]
_parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
ci_url = "%s://%s" % (_parsed_project_url.scheme,
_parsed_project_url.netloc)
gitlab_token = os.environ["GITLAB_TOKEN"]
gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
gitlab_client.auth()
project = gitlab_client.projects.get(project_id)
job = project.jobs.get(job_id)
return job.attributes["status"] == "running"
def __init__(self, bot):
NotifyModule.__init__(self,
bot,
name="gitlab",
desc="Gitlab Interface",
delay=60)
self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
self.gl.auth()
# DEBUG
p = GitLabProject(id=31)
pr = self.gl.projects.get(31)
p.update_from_gitlab(pr)
self.bot.session.add(p)
self.bot.session.commit()
def get_credentials(self, username, password, delete_repo=False):
try:
g = Gitlab(self.gitlab_hostname)
rv = g.login(username, password)
if not rv:
return None, False
else:
return g.headers["PRIVATE-TOKEN"], True
except gitlab.exceptions.HttpError as he:
if str(he) == "401 Unauthorized":
return None, False
else:
raise ChisubmitException(
"Unexpected error getting authorization token (Reason: %s)"
% (he), he)
class TestGitlabV91(TestCase):
def setUp(self):
self.user = os.environ.get('gitlab_user', 'root')
self.password = os.environ.get('gitlab_password', '5iveL!fe')
self.host = os.environ.get('gitlab_host', 'http://*****:*****@example.com')
log_to_term('create_user', create_user)
delete_user = self.gitlab.deleteuser(create_user['id'])
log_to_term('delete_user', delete_user)
def set_project(self, project=None, url=None, token=None):
self.project_name = project if project is not None else self.project_name
self.url = url if url is not None else self.url
self.token = token if token is not None else self.token
if not (self.project_name and self.url and self.token):
raise ValueError(
'project, url, and token are required in order to set project!'
)
self.gl = Gitlab(url, token)
self.gl.auth()
self.project = self.gl.projects.get(self.project_name)
def get_gitlab_project(group, subgroup):
project = get_project_for_group(group, subgroup)
if project is None:
return None
gl = Gitlab.from_config(_gitlab_conf_section, [_gitlab_conf_file])
return gl.projects.get(project)
def get_gitlab_client(credentials: dict = None,
host: str = None,
**kwargs: Any) -> "Gitlab":
"""
Utility function for loading gitlab client objects from a given set of credentials.
Args:
- credentials (dict, optional): a dictionary of AWS credentials used to
initialize the Client; if not provided, will attempt to load the
Client using ambient environment settings
- host (str, optional): the host string for gitlab server users. If not provided, defaults
to https://gitlab.com
- **kwargs (Any, optional): additional keyword arguments to pass to the gitlab Client
Returns:
- Client: an initialized and authenticated gitlab Client
"""
if not Gitlab:
raise ImportError(
"Unable to import Gitlab, please ensure you have installed the gitlab extra"
)
if credentials:
access_token = credentials.get("GITLAB_ACCESS_TOKEN")
else:
access_token = prefect.context.get("secrets",
{}).get("GITLAB_ACCESS_TOKEN", None)
if not access_token:
access_token = os.getenv("GITLAB_ACCESS_TOKEN", None)
if not host:
host = "https://gitlab.com"
return Gitlab(host, private_token=access_token, **kwargs)
def configure(gl_base_url: str, token: str, secret: str, bot_base_url: str,
repo_options: RepoOptions) -> None:
"""Configure webhooks in GitLab repo."""
events = {
'push_events': False,
'issues_events': False,
'confidential_issues_events': False,
'merge_requests_events': False,
'tag_push_events': False,
'note_events': False,
'job_events': False,
'pipeline_events': repo_options.gl_auto_cancel_pipelines,
'wiki_page_events': False
}
hook_url = urljoin(bot_base_url, GITLAB_ENDPOINT)
gitlab = Gitlab(gl_base_url, private_token=token)
project = gitlab.projects.get(repo_options.gl_repo_path)
hooks = project.hooks.list()
hook = _find(hooks, hook_url)
if events['pipeline_events'] is True:
if hook is None:
_create(project, hook_url, secret, events)
else:
events_is_equal = True
for event, value in events.items():
events_is_equal &= hook.attributes[event] == value
if not events_is_equal:
_update(hook, secret, events, repo_options.gl_repo_path)
else:
if hook is not None:
_delete(hook, repo_options.gl_repo_path)
class RepoCreator:
def __init__(self, gitlab_url, gitlab_token, project_name, namespace, description, jenkins_server,
jenkins_username, jenkins_password, hipchat_server, room_id, room_token):
self.gitlab = Gitlab(gitlab_url, token=gitlab_token)
self.project_name = project_name
self.namespace = namespace
self.description = description
self.jenkins_server = jenkins_server
self.jenkins_project_name = project_name
self.jenkins_username = jenkins_username
self.jenkins_password = jenkins_password
self.hipchat_server = hipchat_server
self.room_id = room_id
self.room_token = room_token
def make_new_project(self):
self.gitlab.create_projcet(self.project_name, self.description, self.namespace)
self.activate_services()
def activate_services(self):
project_id = self.gitlab.get_projcet_id_by_name(self.project_name)
self.gitlab.activate_hipchat(project_id, self.hipchat_server, self.room_id, self.room_token)
self.gitlab.activate_jenkins(project_id, self.jenkins_server, self.project_name, self.jenkins_username,
self.jenkins_password)
def crear_repo(package_name):
md5hash_pn = md5(package_name).hexdigest()
first_pref = md5hash_pn[0:2]
second_pref = md5hash_pn[2:4]
workingdir = root_git_dir+"/"+first_pref+"/"+second_pref+"/" + package_name
if not(os.access(root_git_dir+"/"+first_pref, os.F_OK)):
os.mkdir(root_git_dir+"/"+first_pref)
os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
elif not(os.access(root_git_dir+"/"+first_pref+"/"+second_pref, os.F_OK)):
os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
repo = pygit2.init_repository(workingdir)
dashed_package_name=package_name.replace('.','-').lower()
myRemote = repo.remotes.create(package_name, gitlab_url+'/marvin/'+dashed_package_name+'.git')
gl = Gitlab (gitlab_url, gitlab_token)
gl.auth()
p = gl.Project({'name': package_name, 'public':True})
p.save()
return repo
def GitLab_Work_Log(url, token):
gl = Gitlab(url, token)
gl.auth()
work_log = ''
for project in gl.Project():
work_log += "Project Name: " + project.name + '\n'
for commit in project.Commit():
if commit.author_email == gl.user.email:
commited_at = datetime.strptime(commit.created_at[:19], '%Y-%m-%dT%H:%M:%S')
if datetime.today().date()-timedelta(1) == commited_at.date():
work_log += commit.message + '\n'
for merge in project.MergeRequest():
if merge.author.id == gl.user.id:
merged_at = datetime.strptime(merge.created_at, '%Y-%m-%dT%H:%M:%S.%fZ')
if datetime.today().date()-timedelta(1) == merged_at.date():
work_log += merge.title + '\n'
return work_log
class GitLabModule(NotifyModule):
"""A module to follow gitlab's projects"""
_config = (("url", str, ""), ("token", str, ""),
("ssl_verify", bool, True), ("max_commits", int, 10))
def __init__(self, bot):
NotifyModule.__init__(self,
bot,
name="gitlab",
desc="Gitlab Interface",
delay=60)
self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
self.gl.auth()
# DEBUG
p = GitLabProject(id=31)
pr = self.gl.projects.get(31)
p.update_from_gitlab(pr)
self.bot.session.add(p)
self.bot.session.commit()
def do_action(self):
issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()}
for prj in self.bot.session.query(GitLabProject).all():
gl_prj = self.gl.projects.get(prj.id)
commits = gl_prj.commits.list(per_page=self.max_commits)
last = commits[0].id
while commits and commits[0].id != prj.last_commit:
commit = commits.pop(0)
self.bot.say(MSG['commit'] % (prj.name, commit.author_name, commit.title))
prj.last_commit = last
for issue in gl_prj.issues.list(all=True):
if issue.id not in issues.keys():
issues[issue.id] = GitLabIssue(id=issue.id)
self.bot.session.add(issues[issue.id])
if issue.state == 'opened':
self.bot.say(MSG['issue'] % (prj.name, issue.title, prj.web_url, issue.iid))
issues[issue.id].update_from_gitlab(issue)
self.bot.session.commit()
def borrar_repo(package_name):
filepath = repo_name(package_name)
gl = Gitlab (gitlab_url, gitlab_token)
gl.auth()
# Project search no anda bien, da error si pongo el package_name entero
# Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar
# asi que buscamos por la ultima palabra del nombre (esperando que no sea Android)
# e iteramos sobre los resultados hasta encontrar package_name
split_name = package_name.split('.')
searchterm = split_name[len(split_name)-1]
projlist = gl.search_projects(searchterm)
for project in projlist:
if project.name == package_name:
break
else:
project = None
if project == None:
raise Exception("El proyecto no existe en GitLab")
else:
project.delete()
rmtree(filepath)
def __init__(self, gitlab_url, gitlab_token, project_name, namespace, description, jenkins_server,
jenkins_username, jenkins_password, hipchat_server, room_id, room_token):
self.gitlab = Gitlab(gitlab_url, token=gitlab_token)
self.project_name = project_name
self.namespace = namespace
self.description = description
self.jenkins_server = jenkins_server
self.jenkins_project_name = project_name
self.jenkins_username = jenkins_username
self.jenkins_password = jenkins_password
self.hipchat_server = hipchat_server
self.room_id = room_id
self.room_token = room_token
def check_gitlab_repo(project_id=None):
"""
:param project_id:
:return:
"""
new_items = []
gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
gl.auth()
nxsq = gl.projects.get(project_id)
key = 'antbs:monitor:gitlab:{0}'.format(project_id)
last_updated = db.get(key)
events = nxsq.events.list()
for event in events:
if event.action_name == 'pushed to':
if event.created_at != last_updated:
db.set(key, event.created_at)
new_items = ['numix-icon-theme-square']
break
return new_items
def __init__(self, bot):
NotifyModule.__init__(self,
bot,
name="gitlab",
desc="Gitlab Interface",
delay=60)
self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
self.gl.auth()
# DEBUG
p = GitLabProject(id=31)
pr = self.gl.projects.get(31)
p.update_from_gitlab(pr)
self.bot.session.add(p)
self.bot.session.commit()
def connect_to_gitlab(gitlab_hostname, username, password, gitlab_ssl = True, verify_ssl = True):
log("Connecting to GitLab")
if not gitlab_ssl:
hostname = "http://{}".format(gitlab_hostname)
else:
hostname = "https://{}".format(gitlab_hostname)
try:
g = Gitlab(hostname, verify_ssl = verify_ssl)
rv = g.login(username, password)
if not rv:
error("Could not connect to Git server ({}). Reason unknown.".format(gitlab_hostname))
else:
return g
except requests.exceptions.SSLError:
error_msg = "Your computer is not set up to trust the CS department's SSL certificate.\n"
error_msg += "Try running the setup script with the --skip-ssl-verify option."
error(error_msg)
except HttpError as he:
if str(he) == "401 Unauthorized":
error("Could not connect to Git server (incorrect username/password)")
else:
error("Unexpected error while connecting to Git server (Reason: {})".format(he))
def auth(**connection_args):
'''
Set up gitlab credentials
Only intended to be used within Gitlab-enabled modules
'''
prefix = "gitlab."
# look in connection_args first, then default to config file
def get(key, default=None):
return connection_args.get('connection_' + key,
__salt__['config.get'](prefix + key, default))
user = get('user', 'admin')
password = get('password', 'ADMIN')
token = get('token')
url = get('url', 'https://localhost/')
if token:
git = Gitlab(url, token=token)
else:
git = Gitlab(url)
git.login(user, password)
return git
def check_gitlab_repo(project_id=None):
"""
:param project_id:
:return:
"""
new_items = []
gl = Gitlab("https://gitlab.com", GITLAB_TOKEN)
gl.auth()
nxsq = gl.Project(id=project_id)
key = "antbs:monitor:gitlab:%s" % project_id
last_updated = db.get(key)
events = nxsq.Event()
for event in events:
if event.action_name == "pushed to":
if event.created_at != last_updated:
db.set(key, event.created_at)
new_items.append(["numix-icon-theme-square"])
new_items.append(["numix-icon-theme-square-kde"])
break
return new_items
def check_for_new_items():
db.set('FEED_CHECKED', 'True')
db.expire('FEED_CHECKED', 900)
new_items = []
gh = login(token=GITHUB_TOKEN)
last_id = db.get('ANTBS_GITHUB_LAST_EVENT') or ''
repo = gh.repository('numixproject', "numix-icon-theme")
commits = repo.commits()
latest = None
try:
commit = commits.next()
latest = commit.sha
except StopIteration:
pass
if latest != last_id:
db.set('ANTBS_GITHUB_LAST_EVENT', latest)
new_items.append(['numix-icon-theme'])
gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
gl.auth()
nxsq = gl.Project(id='61284')
last_updated = db.get('ANTBS_GITLAB_LAST_UPDATED')
events = nxsq.Event()
for event in events:
if event.action_name == 'pushed to':
if event.created_at != last_updated:
db.set('ANTBS_GITLAB_LAST_UPDATED', event.created_at)
new_items.append(['numix-icon-theme-square'])
new_items.append(['numix-icon-theme-square-kde'])
break
if len(new_items) > 0:
add_to_build_queue(new_items)
def connect(self, credentials):
# Credentials are a GitLab private token
self.gitlab = Gitlab(self.gitlab_hostname, token=credentials, verify_ssl=self.ssl_verify)
try:
# Test connection by grabbing current user
user = self.gitlab.currentuser()
if "message" in user and user["message"] == "401 Unauthorized":
raise ChisubmitException("Invalid GitLab credentials for server '%s'" % (self.gitlab_hostname))
if "username" not in user:
raise ChisubmitException("Unexpected error connecting to GitLab server '%s'" % (self.gitlab_hostname))
except Exception as e:
raise
raise ChisubmitException("Unexpected error connecting to GitLab server '%s': %s" % (self.gitlab_hostname, e))
class GitlabClient(object):
"""Simple Gitlab client."""
def __init__(self, url='https://gitlab.com'):
"""Initialize the Gitlab client.
:param url: Base URL of the Gitlab server (https://gitlab.com by default)
"""
self._url = url
self._gitlab = None
def login(self, token):
"""Login to Gitlab.
:param token: The user's Gitlab private token.
"""
self._gitlab = Gitlab(self._url, token)
def get_projects(self):
"""Fetch the projects owned by the user.
:return list: of projects.
"""
return self._gitlab.owned_projects(per_page=1000)
def get_project(self, name=None):
"""Return the project with the given name.
:param name: Name of the project to return.
:raise NotFound: if the project does not exist.
"""
if not name:
name = self.get_project_name()
projects = self.get_projects()
for p in projects:
if p.name == name:
return p
raise NotFound(name)
def create_project(self, name, wiki_enabled=False, public=False):
"""Create a project.
:param name: Name of the project.
:param wiki_enabled: Enable the wiki for this project.
:param public: Make the project public.
:return dict: with the created project
"""
try:
self.get_project(name)
except NotFound:
pass
else:
raise GitlabException("Project {0} already exists.".format(name))
project = self._gitlab.Project({'name': name, 'wiki_enabled': wiki_enabled,
'public': public})
project.save()
return project
def track(self, project_name='gitlab', branch='master',
remote_name='gitlab', no_push=False):
"""Set a gitlab repository as remote for the current git checkout.
:return: The gitlab git remote.
"""
project = self.get_project(project_name)
repo = Repo('.')
if not remote_name:
raise GitlabException('Invalid remote name {0}'.format(remote_name))
try:
self.get_remote(remote_name)
except NotFound:
pass
else:
raise GitlabException('Remote name {0} already exists.'.format(remote_name))
remote = repo.create_remote(remote_name, project.ssh_url_to_repo)
remote.push(branch, set_upstream=True)
return remote
def get_gitlab_remote(self):
"""Return the gitlab remote of the repository in the current directory.
:raise NotFound: if the gitlab remote is not found.
"""
return self.get_remote('gitlab')
def get_remote(self, name):
"""Return the remote with the given name of the repository in the current directory."""
repo = Repo('.')
if not hasattr(repo, 'remotes'):
raise NotFound()
for remote in repo.remotes:
if remote.name == name:
return remote
raise NotFound()
def get_project_name(self):
"""Return the name of the gitlab project that is tracking the repository in the current directory."""
remote = self.get_gitlab_remote()
return self.get_project_name_from_url(remote.url)
@staticmethod
def get_project_name_from_url(url):
return re.search(r'/(\S+).git', url).groups()[0]
def get_project_page(self, name=None):
"""Return the url of the page of a Gitlab project.
:param name: Name of the project. If not provided, it will use the project name
tracking the repository in the current directory.
:return: Gitlab project page url.
"""
project = self.get_project(name)
url = project.http_url_to_repo
if url.endswith('.git'):
url = url[:-4]
return url
def main(argv=None):
'''
Process the command line arguments and create the JSON dump.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Export all users/issues from GitLab to JIRA JSON format.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('gitlab_url',
help='The full URL to your GitLab instance.')
parser.add_argument('-d', '--date_filter',
help='Only include issues, notes, etc. created after\
the specified date. Expected format is \
YYYY-MM-DD',
type=get_datetime, default='1970-01-01')
parser.add_argument('-e', '--include_empty',
help='Include projects in output that do not have any\
issues.',
action='store_true')
parser.add_argument('-i', '--ignore_list',
help='List of project names to exclude from dump.',
type=argparse.FileType('r'))
parser.add_argument('-p', '--password',
help='The password to use to authenticate if token is \
not specified. If password and token are both \
unspecified, you will be prompted to enter a \
password.')
parser.add_argument('-P', '--page_size',
help='When retrieving result from GitLab, how many \
results should be included in a given page?.',
type=int, default=20)
parser.add_argument('-s', '--verify_ssl',
help='Enable SSL certificate verification',
action='store_true')
parser.add_argument('-t', '--token',
help='The private GitLab API token to use for \
authentication. Either this or username and \
password must be set.')
parser.add_argument('-u', '--username',
help='The username to use for authentication, if token\
is unspecified.')
parser.add_argument('-v', '--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0, action='count')
parser.add_argument('--version', action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
args.page_size = max(100, args.page_size)
# Convert verbose flag to actually logging level
log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
log_level = log_levels[min(args.verbose, 2)]
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'), level=log_level)
# Setup authenticated GitLab instance
if args.token:
git = GitLab(args.gitlab_url, token=args.token,
verify_ssl=args.verify_ssl)
else:
if not args.username:
print('Username: '******'').strip()
if not args.password:
args.password = getpass.getpass('Password: '******'Creating project entries...', end="", file=sys.stderr)
sys.stderr.flush()
key_set = set()
mentioned_users = set()
if args.ignore_list is not None:
ignore_list = {line.strip().lower() for line in args.ignore_list}
else:
ignore_list = {}
for project in gen_all_results(git.getallprojects,
per_page=args.page_size):
proj_name_lower = project['name'].lower()
if proj_name_lower not in ignore_list and project['issues_enabled']:
project_issues = []
for issue in gen_all_results(git.getprojectissues, project['id'],
per_page=args.page_size):
if args.date_filter < datetime.strptime(issue['updated_at'],
TIME_FORMAT):
project_issues.append(issue)
else:
for note in git.getissuewallnotes(project['id'],
issue['id']):
if (args.date_filter <
datetime.strptime(note['created_at'],
TIME_FORMAT)):
project_issues.append(issue)
break
if project_issues or args.include_empty:
jira_project = {}
jira_project['name'] = project['name_with_namespace']
key = project['name']
if key.islower():
key = key.title()
key = re.sub(r'[^A-Z]', '', key)
if len(key) < 2:
key = re.sub(r'[^A-Za-z]', '',
project['name'])[0:2].upper()
added = False
suffix = 65
while key in key_set:
if not added:
key += 'A'
else:
suffix += 1
key = key[:-1] + chr(suffix)
key_set.add(key)
jira_project['key'] = key
jira_project['description'] = md_to_wiki(project['description'])
# jira_project['created'] = project['created_at']
jira_project['issues'] = []
for issue in project_issues:
jira_issue = {}
jira_issue['externalId'] = issue['iid']
if issue['state'] == 'closed':
jira_issue['status'] = 'Closed'
jira_issue['resolution'] = 'Resolved'
else:
jira_issue['status'] = 'Open'
jira_issue['description'] = md_to_wiki(issue['description'])
jira_issue['reporter'] = issue['author']['username']
mentioned_users.add(jira_issue['reporter'])
jira_issue['labels'] = issue['labels']
jira_issue['summary'] = issue['title']
if issue['assignee']:
jira_issue['assignee'] = issue['assignee']['username']
mentioned_users.add(jira_issue['assignee'])
jira_issue['issueType'] = 'Bug'
jira_issue['comments'] = []
# Get all comments/notes
for note in git.getissuewallnotes(project['id'],
issue['id']):
jira_note = {}
jira_note['body'] = md_to_wiki(note['body'])
jira_note['author'] = note['author']['username']
mentioned_users.add(jira_note['author'])
jira_note['created'] = note['created_at']
jira_issue['comments'].append(jira_note)
jira_project['issues'].append(jira_issue)
output_dict['projects'].append(jira_project)
print('.', end="", file=sys.stderr)
sys.stderr.flush()
print('\nCreating user entries...', end="", file=sys.stderr)
sys.stderr.flush()
for user in gen_all_results(git.getusers, per_page=args.page_size):
# Only add users who are actually referenced in issues
if user['username'] in mentioned_users:
jira_user = {}
jira_user['name'] = user['username']
jira_user['fullname'] = user['name']
jira_user['email'] = user['email']
jira_user['groups'] = ['gitlab-users']
jira_user['active'] = (user['state'] == 'active')
output_dict['users'].append(jira_user)
print('.', end="", file=sys.stderr)
sys.stderr.flush()
print('\nPrinting JSON output...', file=sys.stderr)
sys.stderr.flush()
print(json.dumps(output_dict, indent=4))
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def pgreen(t):
print(OKGREEN + t + ENDC)
if __name__ == '__main__':
gl = Gitlab.from_config(None, [os.path.expanduser("~/.gitlab.cfg")])
parser = argparse.ArgumentParser()
parser.add_argument("action")
args = parser.parse_args()
if os.getenv('BLOCK_BUTTON'):
if args.action == "todo":
call('browse https://gitlabee.dt.renault.com/dashboard/todos'.split(' '))
elif args.action == "issues":
call('browse https://gitlabee.dt.renault.com/dashboard/issues?assignee_id=248'.split(' '))
elif args.action == "mrs":
call('browse https://gitlabee.dt.renault.com/dashboard/merge_requests?assignee_id=248'.split(' '))
if args.action == "todo":
nb = 0
__author__ = 'ricard'
from redmine import Redmine
from gitlab import Gitlab
from configparser import ConfigParser
config = ConfigParser()
config.read_file(open('defaults.cfg'))
redmine = Redmine(config.get('redmine', 'url'), key=config.get('redmine', 'key') )
red_project = redmine.project.get(config.get('redmine', 'project'))
gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
gl.auth()
look_for = config.get('gitlab', 'project')
for p in gl.Project(per_page=1000):
# print(p.path_with_namespace)
if p.path_with_namespace == look_for:
gl_project_id = p.id
gl_project = p
break
print(gl_project.id)
closed_status = []
for status in redmine.issue_status.all():
# print(status, list(status))
if getattr(status, 'is_closed', False):
closed_status.append(status.id)
trackers = {
def do_poll(self):
"""Poll Jenkins Server to get Project Information"""
while True:
# Get a list of ALL Jobs on Jenkins
for job_name in self.jenk_client.keys():
jobDetail = self.jenk_client[job_name]
lastBuild = jobDetail.get_last_build()
buildInfo = BuildItem()
if lastBuild.is_running():
buildInfo["status"] = "STARTED"
else:
buildInfo["status"] = lastBuild.get_status()
buildInfo["timestamp"] = lastBuild.get_timestamp().isoformat()
buildInfo["label"] = lastBuild.name
buildInfo["cause"] = "" # Need a clean way to retrieve this value
# TODO: swap GitLab specific library for Git Library
gl_client = Gitlab(self.options["gitlab"], self.options["gitlab_key"])
gl_client.auth()
### complete hack....
try:
proj_url = jobDetail.get_scm_url()[0]
except Exception:
continue
sep = proj_url.rfind("/")
start = proj_url.rfind("/", 0, sep-1)
proj_name = proj_url[start+1:]
project = None
for proj in gl_client.Project():
if proj.path_with_namespace == proj_name:
project = proj
if project is None:
continue # TODO raise error
### TODO remove the above ... soon
commit = project.Commit(lastBuild.get_revision())
repoInfo = RepoItem()
repoInfo["branch"] = jobDetail.get_scm_branch()[0]
repoInfo["commit"] = commit.id
repoInfo["blame"] = commit.author_name
repoInfo["timestamp"] = commit.created_at
### TODO Refactor to "JobsList" from ProjectsList
pushJob = None
try:
## We use this order, since it will be in an infinite
## loop, the likely scenario is that we will just be
## updating the status of jobs
pushJob = PROJECTS.update(job_name, buildInfo, repoInfo)
except IndexError:
pushJob = PROJECTS.add(job_name, buildInfo, repoInfo)
if pushJob is not None:
WEBSOCK_DO_PUSH_EVENT.set(push=pushJob)
def connect_to_gitlab():
gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
gl.auth()
return gl
def main(argv=None):
'''
Process the command line arguments and create the JSON dump.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Transfer all projects/repositories from GitLab to Stash. \
Note: This script assumes you have your SSH key \
registered with both GitLab and Stash.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('gitlab_url',
help='The full URL to your GitLab instance.')
parser.add_argument('stash_url',
help='The full URL to your Stash instance.')
parser.add_argument('-p', '--password',
help='The password to use to authenticate if token is \
not specified. If password and token are both \
unspecified, you will be prompted to enter a \
password.')
parser.add_argument('-P', '--page_size',
help='When retrieving result from GitLab, how many \
results should be included in a given page?.',
type=int, default=20)
parser.add_argument('-s', '--verify_ssl',
help='Enable SSL certificate verification',
action='store_true')
parser.add_argument('-S', '--skip_existing',
help='Do not update existing repositories and just \
skip them.',
action='store_true')
parser.add_argument('-t', '--token',
help='The private GitLab API token to use for \
authentication. Either this or username and \
password must be set.')
parser.add_argument('-u', '--username',
help='The username to use for authentication, if token\
is unspecified.')
parser.add_argument('-v', '--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0, action='count')
parser.add_argument('--version', action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
args.page_size = max(100, args.page_size)
# Convert verbose flag to actually logging level
log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
log_level = log_levels[min(args.verbose, 2)]
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'), level=log_level)
# Setup authenticated GitLab and Stash instances
if args.token:
git = GitLab(args.gitlab_url, token=args.token,
verify_ssl=args.verify_ssl)
else:
git = None
if not args.username:
print('Username: '******'').strip()
if not args.password:
args.password = getpass.getpass('Password: '******'Retrieving existing Stash projects...', end="", file=sys.stderr)
sys.stderr.flush()
key_set = {proj['key'] for proj in stash.projects}
stash_project_names = {proj['name'] for proj in stash.projects}
names_to_keys = {proj['name']: proj['key'] for proj in stash.projects}
print('done', file=sys.stderr)
sys.stderr.flush()
updated_projects = set()
repo_to_slugs = {}
failed_to_clone = set()
cwd = os.getcwd()
transfer_count = 0
skipped_count = 0
print('Processing GitLab projects...', file=sys.stderr)
sys.stderr.flush()
for project in gen_all_results(git.getallprojects,
per_page=args.page_size):
print('\n' + ('=' * 80) + '\n', file=sys.stderr)
sys.stderr.flush()
proj_name = project['namespace']['name']
# Create Stash project if it doesn't already exist
if proj_name not in stash_project_names:
# Create Stash project key
key = proj_name
if key.islower():
key = key.title()
key = re.sub(r'[^A-Z]', '', key)
if len(key) < 2:
key = re.sub(r'[^A-Za-z]', '', proj_name)[0:2].upper()
added = False
suffix = 65
while key in key_set:
if not added:
key += 'A'
else:
suffix += 1
key = key[:-1] + chr(suffix)
key_set.add(key)
# Actually add the project to Stash
print('Creating Stash project "%s" with key %s...' %
(proj_name, key), end="", file=sys.stderr)
sys.stderr.flush()
stash.projects.create(key, proj_name)
names_to_keys[proj_name] = key
stash_project_names.add(proj_name)
print('done', file=sys.stderr)
sys.stderr.flush()
else:
key = names_to_keys[proj_name]
stash_project = stash.projects[key]
# Initialize maping from repository names to slugs for later
if key not in repo_to_slugs:
repo_to_slugs[key] = {repo['name']: repo['slug'] for repo in
stash_project.repos}
# Create Stash-compatible name for repository
# Repository names are limited to 128 characters.
# They must start with a letter or number and may contain spaces,
# hyphens, underscores and periods
repo_name = project['name']
if not repo_name[0].isalnum():
repo_name = 'A ' + repo_name
repo_name = re.sub(r'[^A-Za-z0-9 _.-]', ' ', repo_name)
if len(repo_name) > 128:
repo_name = repo_name[0:128]
# Add repository to Stash project if it's not already there
if repo_name not in repo_to_slugs[key]:
print('Creating Stash repository "%s" in project "%s"...' %
(repo_name, proj_name), end="", file=sys.stderr)
sys.stderr.flush()
stash_repo = stash_project.repos.create(repo_name)
repo_to_slugs[key][repo_name] = stash_repo['slug']
print('done', file=sys.stderr)
sys.stderr.flush()
elif args.skip_existing:
print('Skipping existing Stash repository "%s" in project "%s"' %
(repo_name, proj_name), file=sys.stderr)
sys.stderr.flush()
skipped_count += 1
continue
else:
print('Updating existing Stash repository "%s" in project "%s"' %
(repo_name, proj_name), file=sys.stderr)
sys.stderr.flush()
repo_slug = repo_to_slugs[key][repo_name]
stash_repo = stash_project.repos[repo_slug].get()
for clone_link in stash_repo['links']['clone']:
if clone_link['name'] == 'ssh':
stash_repo_url = clone_link['href']
break
with tempfile.TemporaryDirectory() as temp_dir:
# Clone repository to temporary directory
print('\nCloning GitLab repository...', file=sys.stderr)
sys.stderr.flush()
try:
subprocess.check_call(['git', 'clone', '--mirror',
project['ssh_url_to_repo'],
temp_dir])
except subprocess.CalledProcessError:
print('Failed to clone GitLab repository. This usually when ' +
'it does not exist.', file=sys.stderr)
failed_to_clone.add(project['name_with_namespace'])
skipped_count += 1
continue
os.chdir(temp_dir)
# Check that repository is not empty
try:
subprocess.check_call(['git', 'log', '--format=oneline', '-1'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError:
print('Repository is empty, so skipping push to Stash.',
file=sys.stderr)
skipped_count += 1
else:
# Change remote to Stash and push
print('\nPushing repository to Stash...', file=sys.stderr)
sys.stderr.flush()
subprocess.check_call(['git', 'remote', 'set-url', 'origin',
stash_repo_url])
subprocess.check_call(['git', 'push', '--mirror'])
transfer_count += 1
os.chdir(cwd)
updated_projects.add(proj_name)
print('\n' + ('=' * 35) + 'SUMMARY' + ('=' * 35), file=sys.stderr)
print('{} repositories transferred.\n'.format(transfer_count),
file=sys.stderr)
print('{} repositories skipped.\n'.format(skipped_count),
file=sys.stderr)
print('Projects created/updated:', file=sys.stderr)
for proj in sorted(updated_projects):
print('\t' + proj, file=sys.stderr)
print('Repositories that we could not clone:', file=sys.stderr)
for repo_name in sorted(failed_to_clone):
print('\t' + repo_name, file=sys.stderr)
class GitLabConnection(RemoteRepositoryConnectionBase):
def __init__(self, connection_string, staging, ssl_verify=True):
RemoteRepositoryConnectionBase.__init__(self, connection_string, staging, ssl_verify)
self.gitlab = None
# Map student id's to GitLab user IDs
self.gitlab_user_id = {}
@staticmethod
def get_server_type_name():
return "GitLab"
@staticmethod
def get_connstr_mandatory_params():
return ["gitlab_hostname"]
@staticmethod
def get_connstr_optional_params():
return ["ldap_uid_template"]
@staticmethod
def supports_user_creation():
return True
def get_credentials(self, username, password, delete_repo = False):
try:
g = Gitlab(self.gitlab_hostname)
rv = g.login(username, password)
if not rv:
return None, False
else:
return g.headers["PRIVATE-TOKEN"], True
except gitlab.exceptions.HttpError as he:
if str(he) == "401 Unauthorized":
return None, False
else:
raise ChisubmitException("Unexpected error getting authorization token (Reason: %s)" % (he), he)
def connect(self, credentials):
# Credentials are a GitLab private token
self.gitlab = Gitlab(self.gitlab_hostname, token=credentials, verify_ssl=self.ssl_verify)
try:
# Test connection by grabbing current user
user = self.gitlab.currentuser()
if "message" in user and user["message"] == "401 Unauthorized":
raise ChisubmitException("Invalid GitLab credentials for server '%s'" % (self.gitlab_hostname))
if "username" not in user:
raise ChisubmitException("Unexpected error connecting to GitLab server '%s'" % (self.gitlab_hostname))
except Exception as e:
raise
raise ChisubmitException("Unexpected error connecting to GitLab server '%s': %s" % (self.gitlab_hostname, e))
def disconnect(self, credentials):
pass
def init_course(self, course, fail_if_exists=True):
group = self.__get_group(course)
group_name = self.__get_group_name(course)
if fail_if_exists and group is not None:
raise ChisubmitException("Course '%s' already has a GitLab group" % group_name)
if group is None:
if self.staging:
course_name = course.name + " - STAGING"
else:
course_name = course.name
group_name = self.__get_group_name(course)
new_group = self.gitlab.creategroup(course_name, group_name)
if isinstance(new_group, gitlab.exceptions.HttpError):
raise ChisubmitException("Could not create group '%s' (%s)" % (self.__get_group_name(course), str(new_group)), new_group)
return True
else:
return False
def deinit_course(self, course):
group = self.__get_group(course)
if group is not None:
rv = self.gitlab.deletegroup(group["id"])
def exists_user(self, course, course_user):
gitlab_username = self._get_user_git_username(course, course_user)
user = self.__get_user_by_username(gitlab_username)
if user is None:
return False
else:
return True
def create_user(self, course, course_user):
if self.ldap_uid_template is None:
raise ChisubmitException("ldap_uid_template has not been set for this course")
if "USER" not in self.ldap_uid_template:
raise ChisubmitException("ldap_uid_template does not include USER: %s" % self.ldap_uid_template)
gitlab_user_username = self._get_user_git_username(course, course_user)
gitlab_user_name = "%s %s" % (course_user.user.first_name, course_user.user.last_name)
gitlab_user_email = course_user.user.email
# Password doesn't actually matter since we use
# LDAP authentication. Just in case, we set it to
# something complicated
gitlab_user_password = ''.join([random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for i in range(25)])
gitlab_extern_uid = self.ldap_uid_template.replace("USER", gitlab_user_username)
self.gitlab.createuser(name = gitlab_user_name,
username = gitlab_user_username,
password = gitlab_user_password,
email = gitlab_user_email,
provider = "ldapmain", # TODO: Make this configurable
confirm = False,
extern_uid = gitlab_extern_uid
)
def update_instructors(self, course):
instructors = course.get_instructors()
usernames = [self._get_user_git_username(course, instructor) for instructor in instructors]
self.__add_users_to_course_group(course, usernames, "owner")
# TODO: Remove instructors that may have been removed
def update_graders(self, course):
graders = course.get_graders()
usernames = [self._get_user_git_username(course, grader) for grader in graders]
self.__add_users_to_course_group(course, usernames, "developer")
# TODO: Remove instructors that may have been removed
def create_team_repository(self, course, team, fail_if_exists=True, private=True):
repo_name = self.__get_team_namespaced_project_name(course, team)
team_members = team.get_team_members()
student_names = ", ".join(["%s %s" % (tm.student.user.first_name, tm.student.user.last_name) for tm in team_members])
repo_description = "%s: Team %s (%s)" % (course.name, team.team_id, student_names)
if not self.staging:
gitlab_students = []
# Make sure users exist
for tm in team_members:
gitlab_student = self.__get_user_by_username(self._get_user_git_username(course, tm.student))
if gitlab_student is None:
raise ChisubmitException("GitLab user '%s' does not exist " % (self._get_user_git_username(course, tm.student)))
gitlab_students.append(gitlab_student)
project = self.__get_team_project(course, team)
if project is not None and fail_if_exists:
raise ChisubmitException("Repository %s already exists" % repo_name)
if project is None:
group = self.__get_group(course)
if group is None:
raise ChisubmitException("Group for course '%s' does not exist" % course.id)
# Workaround: Our GitLab server doesn't like public repositories
#if private:
# public = 0
#else:
# public = 1
gitlab_project = self.gitlab.createproject(team.team_id,
namespace_id = group["id"],
description = repo_description,
public = 0)
if gitlab_project == False:
raise ChisubmitException("Could not create repository %s" % repo_name)
if not self.staging:
for gitlab_student in gitlab_students:
rc = self.gitlab.addprojectmember(gitlab_project["id"],
gitlab_student["id"],
"developer")
if rc == False:
raise ChisubmitException("Unable to add user %s to %s" % (gitlab_student["username"], repo_name))
def update_team_repository(self, course, team):
repo_name = self.__get_team_namespaced_project_name(course, team)
team_members = team.get_team_members()
gitlab_project = self.__get_team_project(course, team)
for tm in team_members:
gitlab_student = self.__get_user_by_username(self._get_user_git_username(course, tm.student))
if gitlab_student is None:
raise ChisubmitException("GitLab user '%s' does not exist " % (self._get_user_git_username(course, tm.student)))
rc = self.gitlab.addprojectmember(gitlab_project["id"],
gitlab_student["id"],
"developer")
if rc == False:
raise ChisubmitException("Unable to add user %s to %s" % (gitlab_student["username"], repo_name))
def exists_team_repository(self, course, team):
repo = self.__get_team_project(course, team)
if repo is None:
return False
else:
return True
def get_repository_git_url(self, course, team):
repo_name = self.__get_team_namespaced_project_name(course, team)
hostname = self.gitlab_hostname.replace("http://","").replace("https://","")
return "git@%s:%s.git" % (hostname, repo_name)
def get_repository_http_url(self, course, team):
repo_name = self.__get_team_namespaced_project_name(course, team)
hostname = self.gitlab_hostname.replace("http://","").replace("https://","")
return "https://%s/%s" % (hostname, repo_name)
def get_commit(self, course, team, commit_sha):
project_api_id = self.__get_team_project_api_id(course, team)
gitlab_commit = self.gitlab.getrepositorycommit(project_api_id, commit_sha)
if gitlab_commit == False:
return None
else:
committer_name = gitlab_commit.get("committer_name", gitlab_commit["author_name"])
committer_email = gitlab_commit.get("committer_email", gitlab_commit["author_email"])
commit = GitCommit(gitlab_commit["id"], gitlab_commit["title"],
gitlab_commit["author_name"], gitlab_commit["author_email"], parse(gitlab_commit["authored_date"]),
committer_name, committer_email, parse(gitlab_commit["committed_date"]))
return commit
def get_latest_commit(self, course, team, branch="master"):
return self.get_commit(course, team, branch)
def create_submission_tag(self, course, team, tag_name, tag_message, commit_sha):
pass
# TODO: Commenting out for now, since GitLab doesn't support updating/removing
# tags through the API
#
# project_name = self.__get_team_namespaced_project_name(course, team)
#
# commit = self.get_commit(course, team, commit_sha)
#
# if commit is None:
# raise ChisubmitException("Cannot create tag %s for commit %s (commit does not exist)" % (tag_name, commit_sha))
#
# rc = self.gitlab.createrepositorytag(project_name, tag_name, commit_sha, tag_message)
# if rc == False:
# raise ChisubmitException("Cannot create tag %s in project %s (error when creating tag)" % (tag_name, project_name))
def update_submission_tag(self, course, team, tag_name, tag_message, commit_sha):
# TODO: Not currently possible with current GitLab API
pass
def get_submission_tag(self, course, team, tag_name):
project_name = self.__get_team_namespaced_project_name(course, team)
gitlab_tag = self.__get_tag(project_name, tag_name)
if gitlab_tag is None:
return None
tag = GitTag(name = gitlab_tag["name"],
commit = self.get_commit(course, team, gitlab_tag["commit"]["id"]))
return tag
def delete_team_repository(self, course, team, fail_if_not_exists):
project_name = self.__get_team_namespaced_project_name(course, team)
project_api_id = self.__get_team_project_api_id(course, team)
repo = self.__get_team_project(course, team)
if repo is None:
if fail_if_not_exists:
raise ChisubmitException("Trying to delete a repository that doesn't exist (%s)" % (project_name))
else:
return
self.gitlab.deleteproject(project_api_id)
def __get_group_name(self, course):
if self.staging:
return course.course_id + "-staging"
else:
return course.course_id
def __get_user_by_username(self, username):
# TODO: Paginations
users = self.gitlab.getusers(search=username)
if users == False:
raise ChisubmitException("Unable to fetch Gitlab users")
if len(users) == 0:
return None
for user in users:
if user["username"] == username:
return user
return None
def __get_group(self, course):
group = self.gitlab.getgroups(group_id = self.__get_group_name(course))
if group == False:
return None
else:
return group
def __get_team_namespaced_project_name(self, course, team):
group_name = self.__get_group_name(course)
s = "%s/%s" % (group_name, team.team_id)
return s.lower()
def __get_team_project_api_id(self, course, team):
project_name = self.__get_team_namespaced_project_name(course, team)
return project_name.replace("/", "%2F")
def __get_team_project(self, course, team):
namespaced_project_name = self.__get_team_namespaced_project_name(course, team)
project = self.gitlab.getproject(namespaced_project_name)
if project == False:
return None
else:
return project
def __add_users_to_course_group(self, course, usernames, access_level):
group_name = self.__get_group_name(course)
group = self.__get_group(course)
if group is None:
raise ChisubmitException("Couldn't add users '%s' to group '%s'. Course group does not exist" % (usernames, group_name))
users = []
for username in usernames:
user = self.__get_user_by_username(username)
if user is None:
raise ChisubmitException("Couldn't add user '%s' to group '%s'. User does not exist" % (username, group_name))
users.append(user)
for user in users:
self.gitlab.addgroupmember(group["id"], user["id"], access_level)
# If the return code is False, we can't distinguish between
# "failed because the user is already in the group" or
# "failed for other reason".
# TODO: Check whether user was actually added to group
def __get_tag(self, project_name, tag_name):
tags = self.gitlab.getrepositorytags(project_name)
if tags == False:
raise ChisubmitException("Couldn't get tags for project %s" % project_name)
for t in tags:
if t["id"] == tag_name:
return t
return None
def __has_tag(self, project_name, tag_name):
return self.__get_tag(project_name, tag_name) is not None
def get_projects(config):
url = config.get('gitlab', 'url')
gl = Gitlab(url, get_token(config))
logger.debug("Connecting to Gitlab {}".format(url))
gl.auth() # XXX catch exceptions
user = gl.user
name = user.name.encode('utf8')
username = user.username.encode('utf8')
logger.info("Connected as {1} ({0})".format(
user.name.encode('utf8'), user.username))
group = gl.Group(config.get('gitlab', 'group')) # XXX catch exceptions
return group.projects