Exemple #1
1
    def login(self, token):
        """Login to Gitlab.

        :param token: The user's Gitlab private token.

        """
        self._gitlab = Gitlab(self._url, token)
    def __init__(self, account):

        id = 0
        for key in j.application.config.getKeysFromPrefix(
                "gitlabclient.server"):
            # key=key.replace("gitlabclient.server.","")
            if key.find("name") <> -1:
                if j.application.config.get(key) == account:
                    key2 = key.replace("gitlabclient.server.", "")
                    id = key2.split(".")[0]
        if id == 0:
            raise RuntimeError("Did not find account:%s for gitlab" % account)
        prefix = "gitlabclient.server.%s" % id
        self.addr = j.application.config.get("%s.addr" % prefix)

        # self.accountPathLocal = j.system.fs.joinPaths("/opt/code",accountName)
        # j.system.fs.createDir(self.accountPathLocal)
        # self._gitlab = gitlab.Gitlab(self.addr)
        login = j.application.config.get("%s.login" % prefix)
        passwd = j.application.config.get("%s.passwd" % prefix)
        self.passwd = passwd
        if passwd <> "":

            Gitlab.__init__(self, self.addr)  #, token=token)
            self.login(login, passwd)
            self.load()
        # for item in dir(self._gitlab):
        #     if item[0]<>"_":
        #         setattr(self,item,getattr(self._gitlab,item))
        self.gitclients = {}

        self.loginName = login
        self.port = 80
Exemple #3
0
    def connect_api(self, max_tries: int = 3):
        token = self.token

        # user gets max_tries tries to get the token correct
        tries = 0
        success = False

        gl = None

        while not success and tries < max_tries:
            try:
                if not (isinstance(token, str) and len(token) > 0):
                    raise Exception('Invalid token')
                gl = Gitlab(url=self.remote_host,
                            private_token=token)  # type: Gitlab
                gl.auth()
                success = True
            except Exception:
                tries += 1
                token = self.set_token()

        if not success:
            print('Could not validate ' + self.TOKEN_NAME + '.')
            quit()

        self.api = gl
Exemple #4
0
    def __init__(self, name, token=None, url=None, ssh=None):
        super(GitLabRepository,
              self).__init__(os.path.join(StateHolder.home_dir, 'gitLab',
                                          name))

        target_url = url if url is not None else "http://gitlab.com"

        if token is None:
            ColorPrint.exit_after_print_messages(
                message="Gitlab configuration is empty in section " + name +
                "!")
        else:
            self.gitlab = Gitlab(target_url, private_token=token)
            self.gitlab.version()  # test connection
            lst = dict()
            projects = self.gitlab.projects.list(membership=True)
            for project in projects:
                project_name = project.name
                lst[project_name] = dict()
                lst[project_name]['git'] = str(project.ssh_url_to_repo)
                if ssh is not None:
                    lst[project_name]['ssh'] = ssh

            self.write_yaml_file(os.path.join(self.target_dir,
                                              'pocok-catalog.yml'),
                                 yaml.dump(data=lst, default_flow_style=False),
                                 create=True)
Exemple #5
0
    def __init__(self):

        FeatureExtractor.__init__(self)

        project = Gitlab('https://jugit.fz-juelich.de').projects.get(3009)
        jsonfiles = [
            f['name'] for f in project.repository_tree()
            if f['type'] == 'blob' and f['name'].endswith('json')
        ]

        minval = maxval = 0
        new_profiles = []
        for jsonfile in jsonfiles:
            f = project.files.get(file_path=jsonfile, ref='master')
            data = json.loads(f.decode())
            src_name = data['name']
            src_info = data['description']
            src_file = jsonfile
            parcellation = parcellations[data['parcellation id']]
            regions_available = data['data']['field names']
            for region in regions_available:
                profile = data['data']['profiles'][region]
                if max(profile) > maxval:
                    maxval = max(profile)
                if min(profile) > minval:
                    minval = min(profile)
                new_profiles.append(
                    ConnectivityProfile(region, profile, regions_available,
                                        src_name, src_info, src_file,
                                        parcellation))

        for profile in new_profiles:
            profile.globalrange = (minval, maxval)
            self.register(profile)
def test_invalid_auth_args():
    with pytest.raises(ValueError):
        Gitlab(
            "http://localhost",
            api_version="4",
            private_token="private_token",
            oauth_token="bearer",
        )
    with pytest.raises(ValueError):
        Gitlab(
            "http://localhost",
            api_version="4",
            oauth_token="bearer",
            http_username="******",
            http_password="******",
        )
    with pytest.raises(ValueError):
        Gitlab(
            "http://localhost",
            api_version="4",
            private_token="private_token",
            http_password="******",
        )
    with pytest.raises(ValueError):
        Gitlab(
            "http://localhost",
            api_version="4",
            private_token="private_token",
            http_username="******",
        )
    def __init__(self, headers):
        self.authenticated = all(
            [header in headers.keys() for header in self.auth_headers])
        if not self.authenticated:
            return
        parsed_id_token = self.parse_jwt_from_headers(headers)
        self.keycloak_user_id = parsed_id_token["sub"]
        self.email = parsed_id_token["email"]
        self.full_name = parsed_id_token["name"]
        self.username = parsed_id_token["preferred_username"]
        self.safe_username = escapism.escape(self.username,
                                             escape_char="-").lower()
        self.oidc_issuer = parsed_id_token["iss"]

        (
            self.git_url,
            self.git_auth_header,
            self.git_token,
        ) = self.git_creds_from_headers(headers)
        self.gitlab_client = Gitlab(
            self.git_url,
            api_version=4,
            oauth_token=self.git_token,
        )
        self.setup_k8s()
Exemple #8
0
    def setUp(self):
        self.user = os.environ.get('gitlab_user', 'root')
        self.password = os.environ.get('gitlab_password', '1WmA1a3ONs9F')
        self.host = os.environ.get('gitlab_host', 'http://localhost:10080')
        self.gitlab = Gitlab(host=self.host, verify_ssl=False)

        self.gitlab.login(user=self.user, password=self.password)
Exemple #9
0
def get_merge_request_overview() -> PullRequestsOverview:
    _prs_to_review: List[PullRequest] = []
    _prs_authored_with_work: List[PullRequest] = []
    _exception = None

    _gl = Gitlab(url=GitlabMrsConfig.GITLAB_HOST,
                 private_token=GitlabMrsConfig.PRIVATE_TOKEN)
    _gl.auth()
    _author_id = _gl.user.id

    mrs = group_mrs(_gl)

    try:
        _prs_to_review: List[PullRequest] = extract_pull_request_data(
            get_merge_requests_to_review(_author_id, mrs))
        _prs_authored_with_work: List[PullRequest] = extract_pull_request_data(
            get_authored_merge_requests(_author_id, mrs))
    except Timeout as e:
        _exception = PullRequestException(GitlabMrsConstants.MODULE,
                                          GitlabMrsConstants.TIMEOUT_MESSAGE,
                                          e, traceback.format_exc())
    except GitlabHttpError as e:
        _exception = PullRequestException(
            GitlabMrsConstants.MODULE, GitlabMrsConstants.CONNECTION_MESSAGE,
            e, traceback.format_exc())
    except Exception as e:
        _exception = PullRequestException(GitlabMrsConstants.MODULE,
                                          GitlabMrsConstants.UNKNOWN_MESSAGE,
                                          e, traceback.format_exc())

    return PullRequestsOverview.create(_prs_to_review, _prs_authored_with_work,
                                       _exception)
Exemple #10
0
def process_step_4(message):
    cursor3 = db.token.find_one({"id": encoder(str(message.chat.id))})
    cur = []
    cursor4 = dict(cursor3)
    for j in cursor4["token"]:
        cur.append(decoder(j))

    if message.text in cur:
        try:
            gl = Gitlab('https://git.iu7.bmstu.ru/',
                        private_token=message.text)
            gl.auth()
            username = gl.user.username
            db.token.find_one_and_update(
                {
                    "id": encoder(str(message.chat.id)),
                    "token": encoder(cur)
                }, {'$set': {
                    "idGitLab": encoder(username)
                }})

        except gitlab.GitlabAuthenticationError:
            st = open('./static/access_denied.webp', 'rb')
            bot.send_sticker(message.chat.id, st)

            bot.send_message(
                message.chat.id,
                "Произошла ошибка при авторизации в GitLab. Проверьте правильность токена",
                parse_mode="html",
                reply_markup=types.ReplyKeyboardRemove())
    else:
        bot.send_message(message.chat.id,
                         "Такого TOKEN нет в твоем списке...",
                         parse_mode="html",
                         reply_markup=types.ReplyKeyboardRemove())
Exemple #11
0
def connect_to_gitlab(gitlab_hostname,
                      username,
                      password,
                      gitlab_ssl=True,
                      verify_ssl=True):
    log("Connecting to GitLab")

    if not gitlab_ssl:
        hostname = "http://{}".format(gitlab_hostname)
    else:
        hostname = "https://{}".format(gitlab_hostname)

    try:
        g = Gitlab(hostname, verify_ssl=verify_ssl)
        rv = g.login(username, password)
        if not rv:
            error(
                "Could not connect to Git server ({}). Reason unknown.".format(
                    gitlab_hostname))
        else:
            return g
    except requests.exceptions.SSLError:
        error_msg = "Your computer is not set up to trust the CS department's SSL certificate.\n"
        error_msg += "Try running the setup script with the --skip-ssl-verify option."
        error(error_msg)
    except HttpError as he:
        if str(he) == "401 Unauthorized":
            error(
                "Could not connect to Git server (incorrect username/password)"
            )
        else:
            error(
                "Unexpected error while connecting to Git server (Reason: {})".
                format(he))
def main(args):
    repoName = str(args.repopath.stem)
    repoOwner = str(Path(str(args.repopath.parent).replace(':', '/')).stem)

    config = loadConfig(args.config)
    assert config, "could not load configuation at {}".format(str(args.config))

    # Gitlab or custom hostname
    gl = Gitlab("https://{}".format(config['hostname']),
                private_token=args.token)
    gl.auth()

    ownerId, isUser = getGitlabOwnerId(gl, args.repopath)

    if not ownerId:
        print("could not find project owner", repoOwner)

    if isUser:
        if ownerId and repoName not in getGitlabUserRepoNames(gl, ownerId):
            createGitlabUserRepo(gl, config, ownerId, repoName, args.repodesc)
        else:
            print(args.repopath, "already exists")
    else:
        if ownerId and repoName not in getGitlabUserGroupNames(gl, ownerId):
            createGitlabGroupRepo(gl, config, ownerId, repoName, args.repodesc)
        else:
            print(args.repopath, "already exists")
Exemple #13
0
    def connect(self):
        self.gl = Gitlab(self.gitlab['endpoint'],
                         self.gitlab['token'],
                         ssl_verify=False)
        self.gl.auth()

        self.kb = Kanboard(self.kanboard['endpoint'], self.kanboard['user'],
                           self.kanboard['token'])
Exemple #14
0
def main():
    pipeline_id = int(os.environ["CI_PIPELINE_ID"])
    project_id = os.environ["CI_PROJECT_ID"]
    branch = os.environ["CI_COMMIT_REF_NAME"]
    _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
    ci_url = "%s://%s" % (_parsed_project_url.scheme,
                          _parsed_project_url.netloc)
    gitlab_token = os.environ["GITLAB_TOKEN"]

    gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
    gitlab_client.auth()

    project = gitlab_client.projects.get(project_id)

    interesting_status = ["running", "pending"]
    futures = []
    with ThreadPoolExecutor(max_workers=2) as executor:
        for status in interesting_status:
            futures.append(
                executor.submit(project.pipelines.list,
                                order_by="id",
                                sort="desc",
                                per_page=1,
                                ref=branch,
                                status=status))
    latest_pipelines = []
    for future in futures:
        latest_pipelines += future.result()
    latest_pipelines = sorted(latest_pipelines,
                              key=lambda key: key.attributes["id"],
                              reverse=True)

    if len(latest_pipelines) == 0:
        print(
            "No running pipelines (has a single job been retried?) - continuing"
        )
        exit(0)
    else:
        latest_pipeline = latest_pipelines[0]

    assert pipeline_id <= latest_pipeline.id

    if pipeline_id < latest_pipeline.id:
        print(
            "Running pipeline (%s) is not the latest (%s) - cancelling self" %
            (pipeline_id, latest_pipeline.id),
            flush=True)
        pipeline = project.pipelines.get(pipeline_id, lazy=True)
        pipeline.cancel()
        # Call does not block until actioned so going into a sleep from which it won't wake up...
        sleep(999)
        assert False
    else:
        print("Running pipeline (%s) is the latest - continuing" %
              (pipeline_id, ))

    exit(0)
Exemple #15
0
 def auth(self):
     self.conn = Gitlab(
         self.url,
         private_token=self.private_token,
         oauth_token=self.oauth_token,
         timeout=self.timeout,
         per_page=self.per_page,
     )
     self.conn.auth()
Exemple #16
0
    def clone(self, repo_name, repo_dst: Path):
        gl = Gitlab(url=self.url, private_token=self.auth_token)
        gl.auth()
        current_user = gl.user
        username = current_user.username

        https_link = f'https://{username}:{self.auth_token}@' + self.url[
            8:] + f"/{username}/{repo_name}.git"
        return git.Repo.clone_from(https_link, str(repo_dst))
Exemple #17
0
    def setUp(self):
        self.user = os.environ.get('gitlab_user', 'root')
        self.password = os.environ.get('gitlab_password', '5iveL!fe')
        self.host = os.environ.get('gitlab_host', 'http://gitlab:80')

        self.gitlab = Gitlab(host=self.host,
                             verify_ssl=False,
                             suppress_http_error=False)

        self.gitlab.login(user=self.user, password=self.password)
Exemple #18
0
 def gitlab_userdata(tokens):
     gh = Gitlab("https://gitlab.com", oauth_token=tokens["access_token"])
     gh.auth()
     gluser = gh.user
     return {
         "id": gluser.id,
         "login": gluser.username,
         "name": gluser.name,
         "avatar_url": gluser.avatar_url,
     }
Exemple #19
0
def get_clone_commands(token, repo_root):
    con = Gitlab("http://gitlab.your.domain", token)
    con.auth()

    for project in con.Project(per_page=200):
        url = project.ssh_url_to_repo

        subdir = url.split(":")[1].split("/")[0]
        cmd = "bash -c '(mkdir -p {repo_root}/{subdir} && cd {repo_root}/{subdir} && git clone {url})'".format(
            **locals())
        yield cmd
Exemple #20
0
    def __init__(self, url: str, token: str, user: str, password: str):
        assert url is None or isinstance(url, str)
        assert user is None or isinstance(user, str)
        assert password is None or isinstance(password, str)
        assert token is None or isinstance(token, str)

        url = url or 'https://gitlab.com/'
        if token:
            self.client = Gitlab(url, private_token=token)
        else:
            self.client = Gitlab(url, email=user, password=password)
            if user and password:
                self.client.auth()
Exemple #21
0
 def get_credentials(self, username, password, delete_repo = False):
     try:
         g = Gitlab(self.gitlab_hostname)
         rv = g.login(username, password)
         if not rv:
             return None, False
         else:
             return g.headers["PRIVATE-TOKEN"], True
     except gitlab.exceptions.HttpError as he:
         if str(he) == "401 Unauthorized":
             return None, False
         else:
             raise ChisubmitException("Unexpected error getting authorization token (Reason: %s)" % (he), he)
def is_ci_job_running(job_id: int):
    project_id = os.environ["CI_PROJECT_ID"]
    _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
    ci_url = "%s://%s" % (_parsed_project_url.scheme, _parsed_project_url.netloc)
    gitlab_token = os.environ["GITLAB_TOKEN"]

    gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
    gitlab_client.auth()

    project = gitlab_client.projects.get(project_id)

    job = project.jobs.get(job_id)
    return job.attributes["status"] == "running"
Exemple #23
0
def _main(args: argparse.Namespace):
    """Pass arguments to respective function."""
    __log__.info('------- Arguments: -------')
    __log__.info('OUTDIR: %s', args.OUTDIR)
    __log__.info('REPOSITORY_LIST: %s', args.REPOSITORY_LIST.name)
    __log__.info('--gitlab-repos-dir: %s', args.gitlab_repos_dir)
    __log__.info('--gitlab-host: %s', args.gitlab_host)
    __log__.info('------- Arguments end -------')

    gitlab = Gitlab(args.gitlab_host, api_version=4)
    gitlab.repository_prefix = args.gitlab_repos_dir

    store_repository_info(args.REPOSITORY_LIST, gitlab, args.OUTDIR)
Exemple #24
0
def gitlab_connect(token=None):
    """Authenticate the GL wrapper with Gitlab.

    Args:
        token (str): The Gitlab token to authenticate with.
            Defaults to: None.
    """
    token = settings.GITLAB_API_TOKEN
    try:
        gitlab_client = Gitlab("https://gitlab.com", private_token=token)
        gitlab_client.auth()
    except GitlabHttpError as e:
        logger.exception(e)
    return gitlab_client
Exemple #25
0
    def setUp(self):
        self.user = os.environ.get('gitlab_user', 'root')
        self.password = os.environ.get('gitlab_password', '1WmA1a3ONs9F')
        self.host = os.environ.get('gitlab_host', 'http://localhost:10080')
        self.gitlab = Gitlab(host=self.host, verify_ssl=False)
        self.gitlab.host = self.host

        responses.add(responses.POST,
                      self.gitlab.api_url + '/session',
                      json={'private_token': 'test'},
                      status=201,
                      content_type='application/json')

        self.gitlab.login(user=self.user, password=self.password)
Exemple #26
0
def is_ci_job_running(job_id: int):
    project_id = os.environ["CI_PROJECT_ID"]
    _parsed_project_url = urlparse(os.environ["CI_PROJECT_URL"])
    ci_url = "%s://%s" % (_parsed_project_url.scheme,
                          _parsed_project_url.netloc)
    gitlab_token = os.environ["GITLAB_TOKEN"]

    gitlab_client = Gitlab(ci_url, gitlab_token, api_version=4)
    gitlab_client.auth()

    project = gitlab_client.projects.get(project_id)

    job = project.jobs.get(job_id)
    return job.attributes["status"] == "running"
Exemple #27
0
    def __init__(self, bot):
        NotifyModule.__init__(self,
                              bot,
                              name="gitlab",
                              desc="Gitlab Interface",
                              delay=60)
        self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
        self.gl.auth()

        # DEBUG
        p = GitLabProject(id=31)
        pr = self.gl.projects.get(31)
        p.update_from_gitlab(pr)
        self.bot.session.add(p)
        self.bot.session.commit()
Exemple #28
0
 def get_credentials(self, username, password, delete_repo=False):
     try:
         g = Gitlab(self.gitlab_hostname)
         rv = g.login(username, password)
         if not rv:
             return None, False
         else:
             return g.headers["PRIVATE-TOKEN"], True
     except gitlab.exceptions.HttpError as he:
         if str(he) == "401 Unauthorized":
             return None, False
         else:
             raise ChisubmitException(
                 "Unexpected error getting authorization token (Reason: %s)"
                 % (he), he)
Exemple #29
0
class TestGitlabV91(TestCase):
    def setUp(self):
        self.user = os.environ.get('gitlab_user', 'root')
        self.password = os.environ.get('gitlab_password', '5iveL!fe')
        self.host = os.environ.get('gitlab_host', 'http://*****:*****@example.com')
        log_to_term('create_user', create_user)

        delete_user = self.gitlab.deleteuser(create_user['id'])
        log_to_term('delete_user', delete_user)
Exemple #30
0
    def set_project(self, project=None, url=None, token=None):
        self.project_name = project if project is not None else self.project_name
        self.url = url if url is not None else self.url
        self.token = token if token is not None else self.token

        if not (self.project_name and self.url and self.token):
            raise ValueError(
                'project, url, and token are required in order to set project!'
            )

        self.gl = Gitlab(url, token)

        self.gl.auth()

        self.project = self.gl.projects.get(self.project_name)
Exemple #31
0
def get_gitlab_project(group, subgroup):
    project = get_project_for_group(group, subgroup)
    if project is None:
        return None

    gl = Gitlab.from_config(_gitlab_conf_section, [_gitlab_conf_file])
    return gl.projects.get(project)
Exemple #32
0
def get_gitlab_client(credentials: dict = None,
                      host: str = None,
                      **kwargs: Any) -> "Gitlab":
    """
    Utility function for loading gitlab client objects from a given set of credentials.

    Args:
        - credentials (dict, optional): a dictionary of AWS credentials used to
            initialize the Client; if not provided, will attempt to load the
            Client using ambient environment settings
        - host (str, optional): the host string for gitlab server users. If not provided, defaults
            to https://gitlab.com
        - **kwargs (Any, optional): additional keyword arguments to pass to the gitlab Client

    Returns:
        - Client: an initialized and authenticated gitlab Client
    """
    if not Gitlab:
        raise ImportError(
            "Unable to import Gitlab, please ensure you have installed the gitlab extra"
        )

    if credentials:
        access_token = credentials.get("GITLAB_ACCESS_TOKEN")
    else:
        access_token = prefect.context.get("secrets",
                                           {}).get("GITLAB_ACCESS_TOKEN", None)

    if not access_token:
        access_token = os.getenv("GITLAB_ACCESS_TOKEN", None)

    if not host:
        host = "https://gitlab.com"

    return Gitlab(host, private_token=access_token, **kwargs)
Exemple #33
0
def configure(gl_base_url: str, token: str, secret: str, bot_base_url: str,
              repo_options: RepoOptions) -> None:
	"""Configure webhooks in GitLab repo."""
	events = {
		'push_events': False,
		'issues_events': False,
		'confidential_issues_events': False,
		'merge_requests_events': False,
		'tag_push_events': False,
		'note_events': False,
		'job_events': False,
		'pipeline_events': repo_options.gl_auto_cancel_pipelines,
		'wiki_page_events': False
	}
	hook_url = urljoin(bot_base_url, GITLAB_ENDPOINT)
	gitlab = Gitlab(gl_base_url, private_token=token)
	project = gitlab.projects.get(repo_options.gl_repo_path)
	hooks = project.hooks.list()
	hook = _find(hooks, hook_url)
	if events['pipeline_events'] is True:
		if hook is None:
			_create(project, hook_url, secret, events)
		else:
			events_is_equal = True
			for event, value in events.items():
				events_is_equal &= hook.attributes[event] == value
			if not events_is_equal:
				_update(hook, secret, events, repo_options.gl_repo_path)

	else:
		if hook is not None:
			_delete(hook, repo_options.gl_repo_path)
Exemple #34
0
class RepoCreator:
    def __init__(self, gitlab_url, gitlab_token, project_name, namespace, description, jenkins_server,
                 jenkins_username, jenkins_password, hipchat_server, room_id, room_token):
        self.gitlab = Gitlab(gitlab_url, token=gitlab_token)
        self.project_name = project_name
        self.namespace = namespace
        self.description = description

        self.jenkins_server = jenkins_server
        self.jenkins_project_name = project_name
        self.jenkins_username = jenkins_username
        self.jenkins_password = jenkins_password
        self.hipchat_server = hipchat_server
        self.room_id = room_id
        self.room_token = room_token

    def make_new_project(self):
        self.gitlab.create_projcet(self.project_name, self.description, self.namespace)
        self.activate_services()

    def activate_services(self):
        project_id = self.gitlab.get_projcet_id_by_name(self.project_name)
        self.gitlab.activate_hipchat(project_id, self.hipchat_server, self.room_id, self.room_token)
        self.gitlab.activate_jenkins(project_id, self.jenkins_server, self.project_name, self.jenkins_username,
                                     self.jenkins_password)
def crear_repo(package_name):
	md5hash_pn = md5(package_name).hexdigest()
	first_pref = md5hash_pn[0:2]
	second_pref = md5hash_pn[2:4]
	workingdir = root_git_dir+"/"+first_pref+"/"+second_pref+"/" + package_name
	if not(os.access(root_git_dir+"/"+first_pref, os.F_OK)):
		os.mkdir(root_git_dir+"/"+first_pref)
		os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
	elif not(os.access(root_git_dir+"/"+first_pref+"/"+second_pref, os.F_OK)):
		os.mkdir(root_git_dir+"/"+first_pref+"/"+second_pref)
	repo = pygit2.init_repository(workingdir)
	dashed_package_name=package_name.replace('.','-').lower()
	myRemote = repo.remotes.create(package_name, gitlab_url+'/marvin/'+dashed_package_name+'.git')
	gl = Gitlab (gitlab_url, gitlab_token)
	gl.auth()
	p = gl.Project({'name': package_name, 'public':True})
	p.save()
	return repo
def GitLab_Work_Log(url, token):
    gl = Gitlab(url, token)
    gl.auth()
    work_log = ''
    for project in gl.Project():
        work_log += "Project Name: " + project.name + '\n'
        for commit in project.Commit():
            if commit.author_email == gl.user.email:
                commited_at = datetime.strptime(commit.created_at[:19], '%Y-%m-%dT%H:%M:%S')
                if datetime.today().date()-timedelta(1) == commited_at.date():
                    work_log += commit.message + '\n'

        for merge in project.MergeRequest():
            if merge.author.id == gl.user.id:
                merged_at = datetime.strptime(merge.created_at, '%Y-%m-%dT%H:%M:%S.%fZ')
                if datetime.today().date()-timedelta(1) == merged_at.date():
                    work_log += merge.title + '\n'

    return work_log
Exemple #37
0
class GitLabModule(NotifyModule):
    """A module to follow gitlab's projects"""
    _config = (("url", str, ""), ("token", str, ""),
               ("ssl_verify", bool, True), ("max_commits", int, 10))

    def __init__(self, bot):
        NotifyModule.__init__(self,
                              bot,
                              name="gitlab",
                              desc="Gitlab Interface",
                              delay=60)
        self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
        self.gl.auth()

        # DEBUG
        p = GitLabProject(id=31)
        pr = self.gl.projects.get(31)
        p.update_from_gitlab(pr)
        self.bot.session.add(p)
        self.bot.session.commit()

    def do_action(self):
        issues = {i.id: i for i in self.bot.session.query(GitLabIssue).all()}
        for prj in self.bot.session.query(GitLabProject).all():
            gl_prj = self.gl.projects.get(prj.id)
            commits = gl_prj.commits.list(per_page=self.max_commits)
            last = commits[0].id
            while commits and commits[0].id != prj.last_commit:
                commit = commits.pop(0)
                self.bot.say(MSG['commit'] % (prj.name, commit.author_name, commit.title))
            prj.last_commit = last
            for issue in gl_prj.issues.list(all=True):
                if issue.id not in issues.keys():
                    issues[issue.id] = GitLabIssue(id=issue.id)
                    self.bot.session.add(issues[issue.id])
                    if issue.state == 'opened':
                        self.bot.say(MSG['issue'] % (prj.name, issue.title, prj.web_url, issue.iid))
                issues[issue.id].update_from_gitlab(issue)
        self.bot.session.commit()
def borrar_repo(package_name):
	filepath = repo_name(package_name)
	gl = Gitlab (gitlab_url, gitlab_token)
	gl.auth()
	# Project search no anda bien, da error si pongo el package_name entero
	# Si pido todos seria un delirio para la cantidad de proyectos que queremos manejar
	# asi que buscamos por la ultima palabra del nombre (esperando que no sea Android) 
	# e iteramos sobre los resultados hasta encontrar package_name
	split_name = package_name.split('.')

	searchterm = split_name[len(split_name)-1]
	projlist = gl.search_projects(searchterm)
	for project in projlist:
		if project.name == package_name:
			break
	else:
		project = None
	if project == None:
		raise Exception("El proyecto no existe en GitLab")
	else:
		project.delete()
	rmtree(filepath)
Exemple #39
0
    def __init__(self, gitlab_url, gitlab_token, project_name, namespace, description, jenkins_server,
                 jenkins_username, jenkins_password, hipchat_server, room_id, room_token):
        self.gitlab = Gitlab(gitlab_url, token=gitlab_token)
        self.project_name = project_name
        self.namespace = namespace
        self.description = description

        self.jenkins_server = jenkins_server
        self.jenkins_project_name = project_name
        self.jenkins_username = jenkins_username
        self.jenkins_password = jenkins_password
        self.hipchat_server = hipchat_server
        self.room_id = room_id
        self.room_token = room_token
Exemple #40
0
def check_gitlab_repo(project_id=None):
    """

    :param project_id:
    :return:
    """
    new_items = []
    gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
    gl.auth()
    nxsq = gl.projects.get(project_id)
    key = 'antbs:monitor:gitlab:{0}'.format(project_id)
    last_updated = db.get(key)
    events = nxsq.events.list()

    for event in events:
        if event.action_name == 'pushed to':
            if event.created_at != last_updated:
                db.set(key, event.created_at)
                new_items = ['numix-icon-theme-square']

            break

    return new_items
Exemple #41
0
    def __init__(self, bot):
        NotifyModule.__init__(self,
                              bot,
                              name="gitlab",
                              desc="Gitlab Interface",
                              delay=60)
        self.gl = Gitlab(self.url, self.token, ssl_verify=self.ssl_verify)
        self.gl.auth()

        # DEBUG
        p = GitLabProject(id=31)
        pr = self.gl.projects.get(31)
        p.update_from_gitlab(pr)
        self.bot.session.add(p)
        self.bot.session.commit()
Exemple #42
0
def connect_to_gitlab(gitlab_hostname, username, password, gitlab_ssl = True, verify_ssl = True):
    log("Connecting to GitLab")
    
    if not gitlab_ssl:
        hostname = "http://{}".format(gitlab_hostname)
    else:
        hostname = "https://{}".format(gitlab_hostname)

    try:
            g = Gitlab(hostname, verify_ssl = verify_ssl)
            rv = g.login(username, password)
            if not rv:
                error("Could not connect to Git server ({}). Reason unknown.".format(gitlab_hostname))
            else:
                return g
    except requests.exceptions.SSLError:
        error_msg  = "Your computer is not set up to trust the CS department's SSL certificate.\n"
        error_msg += "Try running the setup script with the --skip-ssl-verify option."
        error(error_msg)
    except HttpError as he:
        if str(he) == "401 Unauthorized":
            error("Could not connect to Git server (incorrect username/password)")
        else:
            error("Unexpected error while connecting to Git server (Reason: {})".format(he))
def auth(**connection_args):
    '''
    Set up gitlab credentials

    Only intended to be used within Gitlab-enabled modules
    '''
   
    prefix = "gitlab."

    # look in connection_args first, then default to config file
    def get(key, default=None):
        return connection_args.get('connection_' + key,
            __salt__['config.get'](prefix + key, default))

    user = get('user', 'admin')
    password = get('password', 'ADMIN')
    token = get('token')
    url = get('url', 'https://localhost/')
    if token:
        git = Gitlab(url, token=token)
    else:
        git = Gitlab(url)
        git.login(user, password)
    return git
Exemple #44
0
def check_gitlab_repo(project_id=None):
    """

    :param project_id:
    :return:
    """
    new_items = []
    gl = Gitlab("https://gitlab.com", GITLAB_TOKEN)
    gl.auth()
    nxsq = gl.Project(id=project_id)
    key = "antbs:monitor:gitlab:%s" % project_id
    last_updated = db.get(key)
    events = nxsq.Event()

    for event in events:
        if event.action_name == "pushed to":
            if event.created_at != last_updated:
                db.set(key, event.created_at)
                new_items.append(["numix-icon-theme-square"])
                new_items.append(["numix-icon-theme-square-kde"])

            break

    return new_items
Exemple #45
0
def check_for_new_items():
    db.set('FEED_CHECKED', 'True')
    db.expire('FEED_CHECKED', 900)
    new_items = []
    gh = login(token=GITHUB_TOKEN)
    last_id = db.get('ANTBS_GITHUB_LAST_EVENT') or ''
    repo = gh.repository('numixproject', "numix-icon-theme")
    commits = repo.commits()
    latest = None
    try:
        commit = commits.next()
        latest = commit.sha
    except StopIteration:
        pass

    if latest != last_id:
        db.set('ANTBS_GITHUB_LAST_EVENT', latest)
        new_items.append(['numix-icon-theme'])

    gl = Gitlab('https://gitlab.com', GITLAB_TOKEN)
    gl.auth()
    nxsq = gl.Project(id='61284')
    last_updated = db.get('ANTBS_GITLAB_LAST_UPDATED')
    events = nxsq.Event()

    for event in events:
        if event.action_name == 'pushed to':
            if event.created_at != last_updated:
                db.set('ANTBS_GITLAB_LAST_UPDATED', event.created_at)
                new_items.append(['numix-icon-theme-square'])
                new_items.append(['numix-icon-theme-square-kde'])

            break

    if len(new_items) > 0:
        add_to_build_queue(new_items)
Exemple #46
0
 def connect(self, credentials):
     # Credentials are a GitLab private token
     self.gitlab = Gitlab(self.gitlab_hostname, token=credentials, verify_ssl=self.ssl_verify)    
     try:
         # Test connection by grabbing current user
         user = self.gitlab.currentuser()
         
         if "message" in user and user["message"] == "401 Unauthorized":
             raise ChisubmitException("Invalid GitLab credentials for server '%s'" % (self.gitlab_hostname))
         
         if "username" not in user:
             raise ChisubmitException("Unexpected error connecting to GitLab server '%s'" % (self.gitlab_hostname))
                         
     except Exception as e:
         raise
         raise ChisubmitException("Unexpected error connecting to GitLab server '%s': %s" % (self.gitlab_hostname, e))
Exemple #47
0
class GitlabClient(object):

    """Simple Gitlab client."""

    def __init__(self, url='https://gitlab.com'):
        """Initialize the Gitlab client.

        :param url: Base URL of the Gitlab server (https://gitlab.com by default)

        """
        self._url = url
        self._gitlab = None

    def login(self, token):
        """Login to Gitlab.

        :param token: The user's Gitlab private token.

        """
        self._gitlab = Gitlab(self._url, token)

    def get_projects(self):
        """Fetch the projects owned by the user.

        :return list: of projects.

        """
        return self._gitlab.owned_projects(per_page=1000)

    def get_project(self, name=None):
        """Return the project with the given name.

        :param name: Name of the project to return.
        :raise NotFound: if the project does not exist.

        """
        if not name:
            name = self.get_project_name()
        projects = self.get_projects()
        for p in projects:
            if p.name == name:
                return p
        raise NotFound(name)

    def create_project(self, name, wiki_enabled=False, public=False):
        """Create a project.

        :param name: Name of the project.
        :param wiki_enabled: Enable the wiki for this project.
        :param public: Make the project public.

        :return dict: with the created project

        """
        try:
            self.get_project(name)
        except NotFound:
            pass
        else:
            raise GitlabException("Project {0} already exists.".format(name))
        project = self._gitlab.Project({'name': name, 'wiki_enabled': wiki_enabled,
                                       'public': public})
        project.save()
        return project

    def track(self, project_name='gitlab', branch='master',
              remote_name='gitlab', no_push=False):
        """Set a gitlab repository as remote for the current git checkout.

        :return: The gitlab git remote.

        """
        project = self.get_project(project_name)
        repo = Repo('.')
        if not remote_name:
            raise GitlabException('Invalid remote name {0}'.format(remote_name))
        try:
            self.get_remote(remote_name)
        except NotFound:
            pass
        else:
            raise GitlabException('Remote name {0} already exists.'.format(remote_name))
        remote = repo.create_remote(remote_name, project.ssh_url_to_repo)
        remote.push(branch, set_upstream=True)
        return remote

    def get_gitlab_remote(self):
        """Return the gitlab remote of the repository in the current directory.

        :raise NotFound: if the gitlab remote is not found.

        """
        return self.get_remote('gitlab')

    def get_remote(self, name):
        """Return the remote with the given name of the repository in the current directory."""
        repo = Repo('.')
        if not hasattr(repo, 'remotes'):
            raise NotFound()
        for remote in repo.remotes:
            if remote.name == name:
                return remote
        raise NotFound()

    def get_project_name(self):
        """Return the name of the gitlab project that is tracking the repository in the current directory."""
        remote = self.get_gitlab_remote()
        return self.get_project_name_from_url(remote.url)

    @staticmethod
    def get_project_name_from_url(url):
        return re.search(r'/(\S+).git', url).groups()[0]

    def get_project_page(self, name=None):
        """Return the url of the page of a Gitlab project.

        :param name: Name of the project. If not provided, it will use the project name
            tracking the repository in the current directory.
        :return: Gitlab project page url.

        """
        project = self.get_project(name)
        url = project.http_url_to_repo
        if url.endswith('.git'):
            url = url[:-4]
        return url
def main(argv=None):
    '''
    Process the command line arguments and create the JSON dump.

    :param argv: List of arguments, as if specified on the command-line.
                 If None, ``sys.argv[1:]`` is used instead.
    :type argv: list of str
    '''
    # Get command line arguments
    parser = argparse.ArgumentParser(
        description="Export all users/issues from GitLab to JIRA JSON format.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        conflict_handler='resolve')
    parser.add_argument('gitlab_url',
                        help='The full URL to your GitLab instance.')
    parser.add_argument('-d', '--date_filter',
                        help='Only include issues, notes, etc. created after\
                              the specified date. Expected format is \
                              YYYY-MM-DD',
                        type=get_datetime, default='1970-01-01')
    parser.add_argument('-e', '--include_empty',
                        help='Include projects in output that do not have any\
                              issues.',
                        action='store_true')
    parser.add_argument('-i', '--ignore_list',
                        help='List of project names to exclude from dump.',
                        type=argparse.FileType('r'))
    parser.add_argument('-p', '--password',
                        help='The password to use to authenticate if token is \
                              not specified. If password and token are both \
                              unspecified, you will be prompted to enter a \
                              password.')
    parser.add_argument('-P', '--page_size',
                        help='When retrieving result from GitLab, how many \
                              results should be included in a given page?.',
                        type=int, default=20)
    parser.add_argument('-s', '--verify_ssl',
                        help='Enable SSL certificate verification',
                        action='store_true')
    parser.add_argument('-t', '--token',
                        help='The private GitLab API token to use for \
                              authentication. Either this or username and \
                              password must be set.')
    parser.add_argument('-u', '--username',
                        help='The username to use for authentication, if token\
                              is unspecified.')
    parser.add_argument('-v', '--verbose',
                        help='Print more status information. For every ' +
                             'additional time this flag is specified, ' +
                             'output gets more verbose.',
                        default=0, action='count')
    parser.add_argument('--version', action='version',
                        version='%(prog)s {0}'.format(__version__))
    args = parser.parse_args(argv)

    args.page_size = max(100, args.page_size)

    # Convert verbose flag to actually logging level
    log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
    log_level = log_levels[min(args.verbose, 2)]
    # Make warnings from built-in warnings module get formatted more nicely
    logging.captureWarnings(True)
    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
                                '%(message)s'), level=log_level)

    # Setup authenticated GitLab instance
    if args.token:
        git = GitLab(args.gitlab_url, token=args.token,
                     verify_ssl=args.verify_ssl)
    else:
        if not args.username:
            print('Username: '******'').strip()
        if not args.password:
            args.password = getpass.getpass('Password: '******'Creating project entries...', end="", file=sys.stderr)
    sys.stderr.flush()
    key_set = set()
    mentioned_users = set()
    if args.ignore_list is not None:
        ignore_list = {line.strip().lower() for line in args.ignore_list}
    else:
        ignore_list = {}
    for project in gen_all_results(git.getallprojects,
                                   per_page=args.page_size):
        proj_name_lower = project['name'].lower()
        if proj_name_lower not in ignore_list and project['issues_enabled']:
            project_issues = []
            for issue in gen_all_results(git.getprojectissues, project['id'],
                                         per_page=args.page_size):
                if args.date_filter < datetime.strptime(issue['updated_at'],
                                                        TIME_FORMAT):
                    project_issues.append(issue)
                else:
                    for note in git.getissuewallnotes(project['id'],
                                                      issue['id']):
                        if (args.date_filter <
                                datetime.strptime(note['created_at'],
                                                  TIME_FORMAT)):
                            project_issues.append(issue)
                            break

            if project_issues or args.include_empty:
                jira_project = {}
                jira_project['name'] = project['name_with_namespace']
                key = project['name']
                if key.islower():
                    key = key.title()
                key = re.sub(r'[^A-Z]', '', key)
                if len(key) < 2:
                    key = re.sub(r'[^A-Za-z]', '',
                                 project['name'])[0:2].upper()
                added = False
                suffix = 65
                while key in key_set:
                    if not added:
                        key += 'A'
                    else:
                        suffix += 1
                        key = key[:-1] + chr(suffix)
                key_set.add(key)
                jira_project['key'] = key
                jira_project['description'] = md_to_wiki(project['description'])
                # jira_project['created'] = project['created_at']
                jira_project['issues'] = []
                for issue in project_issues:
                    jira_issue = {}
                    jira_issue['externalId'] = issue['iid']
                    if issue['state'] == 'closed':
                        jira_issue['status'] = 'Closed'
                        jira_issue['resolution'] = 'Resolved'
                    else:
                        jira_issue['status'] = 'Open'

                    jira_issue['description'] = md_to_wiki(issue['description'])
                    jira_issue['reporter'] = issue['author']['username']
                    mentioned_users.add(jira_issue['reporter'])
                    jira_issue['labels'] = issue['labels']
                    jira_issue['summary'] = issue['title']
                    if issue['assignee']:
                        jira_issue['assignee'] = issue['assignee']['username']
                        mentioned_users.add(jira_issue['assignee'])
                    jira_issue['issueType'] = 'Bug'
                    jira_issue['comments'] = []
                    # Get all comments/notes
                    for note in git.getissuewallnotes(project['id'],
                                                      issue['id']):
                        jira_note = {}
                        jira_note['body'] = md_to_wiki(note['body'])
                        jira_note['author'] = note['author']['username']
                        mentioned_users.add(jira_note['author'])
                        jira_note['created'] = note['created_at']
                        jira_issue['comments'].append(jira_note)
                    jira_project['issues'].append(jira_issue)

                output_dict['projects'].append(jira_project)
        print('.', end="", file=sys.stderr)
        sys.stderr.flush()

    print('\nCreating user entries...', end="", file=sys.stderr)
    sys.stderr.flush()
    for user in gen_all_results(git.getusers, per_page=args.page_size):
        # Only add users who are actually referenced in issues
        if user['username'] in mentioned_users:
            jira_user = {}
            jira_user['name'] = user['username']
            jira_user['fullname'] = user['name']
            jira_user['email'] = user['email']
            jira_user['groups'] = ['gitlab-users']
            jira_user['active'] = (user['state'] == 'active')
            output_dict['users'].append(jira_user)
        print('.', end="", file=sys.stderr)
        sys.stderr.flush()

    print('\nPrinting JSON output...', file=sys.stderr)
    sys.stderr.flush()
    print(json.dumps(output_dict, indent=4))
Exemple #49
0
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    def pgreen(t):
        print(OKGREEN + t + ENDC)


if __name__ == '__main__':
    gl = Gitlab.from_config(None, [os.path.expanduser("~/.gitlab.cfg")])
    parser = argparse.ArgumentParser()
    parser.add_argument("action")
    args = parser.parse_args()

    if os.getenv('BLOCK_BUTTON'):
        if args.action == "todo":
            call('browse https://gitlabee.dt.renault.com/dashboard/todos'.split(' '))
        elif args.action == "issues":
            call('browse https://gitlabee.dt.renault.com/dashboard/issues?assignee_id=248'.split(' '))
        elif args.action == "mrs":
            call('browse https://gitlabee.dt.renault.com/dashboard/merge_requests?assignee_id=248'.split(' '))


    if args.action == "todo":
        nb = 0
__author__ = 'ricard'
from redmine import Redmine
from gitlab import Gitlab
from configparser import ConfigParser

config = ConfigParser()
config.read_file(open('defaults.cfg'))

redmine = Redmine(config.get('redmine', 'url'), key=config.get('redmine', 'key') )

red_project = redmine.project.get(config.get('redmine', 'project'))

gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
gl.auth()
look_for = config.get('gitlab', 'project')
for p in gl.Project(per_page=1000):
    # print(p.path_with_namespace)
    if p.path_with_namespace == look_for:
        gl_project_id = p.id
        gl_project = p
        break

print(gl_project.id)

closed_status = []
for status in redmine.issue_status.all():
    # print(status, list(status))
    if getattr(status, 'is_closed', False):
        closed_status.append(status.id)

trackers = {
Exemple #51
0
    def do_poll(self):
        """Poll Jenkins Server to get Project Information"""
        while True:
            # Get a list of ALL Jobs on Jenkins
            for job_name in self.jenk_client.keys():
                jobDetail = self.jenk_client[job_name]
                lastBuild = jobDetail.get_last_build()
                buildInfo = BuildItem()
                if lastBuild.is_running():
                    buildInfo["status"] = "STARTED"
                else:
                    buildInfo["status"] = lastBuild.get_status()
                buildInfo["timestamp"] = lastBuild.get_timestamp().isoformat()
                buildInfo["label"] = lastBuild.name
                buildInfo["cause"] = "" # Need a clean way to retrieve this value


                # TODO: swap GitLab specific library for Git Library
                gl_client = Gitlab(self.options["gitlab"], self.options["gitlab_key"])
                gl_client.auth()

                ### complete hack....
                try:
                    proj_url = jobDetail.get_scm_url()[0]
                except Exception:
                    continue

                sep = proj_url.rfind("/")
                start = proj_url.rfind("/", 0, sep-1)
                proj_name = proj_url[start+1:]
                project = None

                for proj in gl_client.Project():
                    if proj.path_with_namespace == proj_name:
                        project = proj

                if project is None:
                    continue # TODO raise error
                
                ### TODO remove the above ... soon

                commit = project.Commit(lastBuild.get_revision())

                repoInfo = RepoItem()
                repoInfo["branch"] = jobDetail.get_scm_branch()[0]
                repoInfo["commit"] = commit.id
                repoInfo["blame"] = commit.author_name
                repoInfo["timestamp"] = commit.created_at

                ### TODO Refactor to "JobsList" from ProjectsList
                pushJob = None
                try:
                    ## We use this order, since it will be in an infinite
                    ## loop, the likely scenario is that we will just be
                    ## updating the status of jobs
                    pushJob = PROJECTS.update(job_name, buildInfo, repoInfo)
                except IndexError:
                    pushJob = PROJECTS.add(job_name, buildInfo, repoInfo)

                if pushJob is not None:
                    WEBSOCK_DO_PUSH_EVENT.set(push=pushJob)
def connect_to_gitlab():
    gl = Gitlab(config.get('gitlab', 'url'), config.get('gitlab', 'key'))
    gl.auth()
    return gl
def main(argv=None):
    '''
    Process the command line arguments and create the JSON dump.

    :param argv: List of arguments, as if specified on the command-line.
                 If None, ``sys.argv[1:]`` is used instead.
    :type argv: list of str
    '''
    # Get command line arguments
    parser = argparse.ArgumentParser(
        description="Transfer all projects/repositories from GitLab to Stash. \
                     Note: This script assumes you have your SSH key \
                     registered with both GitLab and Stash.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        conflict_handler='resolve')
    parser.add_argument('gitlab_url',
                        help='The full URL to your GitLab instance.')
    parser.add_argument('stash_url',
                        help='The full URL to your Stash instance.')
    parser.add_argument('-p', '--password',
                        help='The password to use to authenticate if token is \
                              not specified. If password and token are both \
                              unspecified, you will be prompted to enter a \
                              password.')
    parser.add_argument('-P', '--page_size',
                        help='When retrieving result from GitLab, how many \
                              results should be included in a given page?.',
                        type=int, default=20)
    parser.add_argument('-s', '--verify_ssl',
                        help='Enable SSL certificate verification',
                        action='store_true')
    parser.add_argument('-S', '--skip_existing',
                        help='Do not update existing repositories and just \
                              skip them.',
                        action='store_true')
    parser.add_argument('-t', '--token',
                        help='The private GitLab API token to use for \
                              authentication. Either this or username and \
                              password must be set.')
    parser.add_argument('-u', '--username',
                        help='The username to use for authentication, if token\
                              is unspecified.')
    parser.add_argument('-v', '--verbose',
                        help='Print more status information. For every ' +
                             'additional time this flag is specified, ' +
                             'output gets more verbose.',
                        default=0, action='count')
    parser.add_argument('--version', action='version',
                        version='%(prog)s {0}'.format(__version__))
    args = parser.parse_args(argv)

    args.page_size = max(100, args.page_size)

    # Convert verbose flag to actually logging level
    log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
    log_level = log_levels[min(args.verbose, 2)]
    # Make warnings from built-in warnings module get formatted more nicely
    logging.captureWarnings(True)
    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
                                '%(message)s'), level=log_level)

    # Setup authenticated GitLab and Stash instances
    if args.token:
        git = GitLab(args.gitlab_url, token=args.token,
                            verify_ssl=args.verify_ssl)
    else:
        git = None
    if not args.username:
        print('Username: '******'').strip()
    if not args.password:
        args.password = getpass.getpass('Password: '******'Retrieving existing Stash projects...', end="", file=sys.stderr)
    sys.stderr.flush()
    key_set = {proj['key'] for proj in stash.projects}
    stash_project_names = {proj['name'] for proj in stash.projects}
    names_to_keys = {proj['name']: proj['key'] for proj in stash.projects}
    print('done', file=sys.stderr)
    sys.stderr.flush()
    updated_projects = set()
    repo_to_slugs = {}
    failed_to_clone = set()
    cwd = os.getcwd()
    transfer_count = 0
    skipped_count = 0
    print('Processing GitLab projects...', file=sys.stderr)
    sys.stderr.flush()
    for project in gen_all_results(git.getallprojects,
                                   per_page=args.page_size):
        print('\n' + ('=' * 80) + '\n', file=sys.stderr)
        sys.stderr.flush()
        proj_name = project['namespace']['name']
        # Create Stash project if it doesn't already exist
        if proj_name not in stash_project_names:
            # Create Stash project key
            key = proj_name
            if key.islower():
                key = key.title()
            key = re.sub(r'[^A-Z]', '', key)
            if len(key) < 2:
                key = re.sub(r'[^A-Za-z]', '', proj_name)[0:2].upper()
            added = False
            suffix = 65
            while key in key_set:
                if not added:
                    key += 'A'
                else:
                    suffix += 1
                    key = key[:-1] + chr(suffix)
            key_set.add(key)

            # Actually add the project to Stash
            print('Creating Stash project "%s" with key %s...' %
                  (proj_name, key), end="", file=sys.stderr)
            sys.stderr.flush()
            stash.projects.create(key, proj_name)
            names_to_keys[proj_name] = key
            stash_project_names.add(proj_name)
            print('done', file=sys.stderr)
            sys.stderr.flush()
        else:
            key = names_to_keys[proj_name]

        stash_project = stash.projects[key]

        # Initialize maping from repository names to slugs for later
        if key not in repo_to_slugs:
            repo_to_slugs[key] = {repo['name']: repo['slug'] for repo in
                                  stash_project.repos}

        # Create Stash-compatible name for repository
        # Repository names are limited to 128 characters.
        # They must start with a letter or number and may contain spaces,
        # hyphens, underscores and periods
        repo_name = project['name']
        if not repo_name[0].isalnum():
            repo_name = 'A ' + repo_name
        repo_name = re.sub(r'[^A-Za-z0-9 _.-]', ' ', repo_name)
        if len(repo_name) > 128:
            repo_name = repo_name[0:128]

        # Add repository to Stash project if it's not already there
        if repo_name not in repo_to_slugs[key]:
            print('Creating Stash repository "%s" in project "%s"...' %
                  (repo_name, proj_name), end="", file=sys.stderr)
            sys.stderr.flush()
            stash_repo = stash_project.repos.create(repo_name)
            repo_to_slugs[key][repo_name] = stash_repo['slug']
            print('done', file=sys.stderr)
            sys.stderr.flush()
        elif args.skip_existing:
            print('Skipping existing Stash repository "%s" in project "%s"' %
                  (repo_name, proj_name), file=sys.stderr)
            sys.stderr.flush()
            skipped_count += 1
            continue
        else:
            print('Updating existing Stash repository "%s" in project "%s"' %
                  (repo_name, proj_name), file=sys.stderr)
            sys.stderr.flush()
            repo_slug = repo_to_slugs[key][repo_name]
            stash_repo = stash_project.repos[repo_slug].get()

        for clone_link in stash_repo['links']['clone']:
            if clone_link['name'] == 'ssh':
                stash_repo_url = clone_link['href']
                break

        with tempfile.TemporaryDirectory() as temp_dir:
            # Clone repository to temporary directory
            print('\nCloning GitLab repository...', file=sys.stderr)
            sys.stderr.flush()
            try:
                subprocess.check_call(['git', 'clone', '--mirror',
                                       project['ssh_url_to_repo'],
                                       temp_dir])
            except subprocess.CalledProcessError:
                print('Failed to clone GitLab repository. This usually when ' +
                      'it does not exist.', file=sys.stderr)
                failed_to_clone.add(project['name_with_namespace'])
                skipped_count += 1
                continue
            os.chdir(temp_dir)

            # Check that repository is not empty
            try:
                subprocess.check_call(['git', 'log', '--format=oneline', '-1'],
                                      stdout=subprocess.DEVNULL,
                                      stderr=subprocess.DEVNULL)
            except subprocess.CalledProcessError:
                print('Repository is empty, so skipping push to Stash.',
                      file=sys.stderr)
                skipped_count += 1
            else:
                # Change remote to Stash and push
                print('\nPushing repository to Stash...', file=sys.stderr)
                sys.stderr.flush()
                subprocess.check_call(['git', 'remote', 'set-url', 'origin',
                                       stash_repo_url])
                subprocess.check_call(['git', 'push', '--mirror'])
                transfer_count += 1

            os.chdir(cwd)

        updated_projects.add(proj_name)


    print('\n' + ('=' * 35) + 'SUMMARY' + ('=' * 35), file=sys.stderr)
    print('{} repositories transferred.\n'.format(transfer_count),
          file=sys.stderr)
    print('{} repositories skipped.\n'.format(skipped_count),
          file=sys.stderr)
    print('Projects created/updated:', file=sys.stderr)
    for proj in sorted(updated_projects):
        print('\t' + proj, file=sys.stderr)
    print('Repositories that we could not clone:', file=sys.stderr)
    for repo_name in sorted(failed_to_clone):
        print('\t' + repo_name, file=sys.stderr)
Exemple #54
0
class GitLabConnection(RemoteRepositoryConnectionBase):

    def __init__(self, connection_string, staging, ssl_verify=True):
        RemoteRepositoryConnectionBase.__init__(self, connection_string, staging, ssl_verify)
        
        self.gitlab = None
        
        # Map student id's to GitLab user IDs
        self.gitlab_user_id = {}

        
    @staticmethod
    def get_server_type_name():
        return "GitLab"
    
    @staticmethod
    def get_connstr_mandatory_params():
        return ["gitlab_hostname"]

    @staticmethod
    def get_connstr_optional_params():
        return ["ldap_uid_template"]
    
    @staticmethod
    def supports_user_creation():
        return True    
    
    def get_credentials(self, username, password, delete_repo = False):
        try:
            g = Gitlab(self.gitlab_hostname)
            rv = g.login(username, password)
            if not rv:
                return None, False
            else:
                return g.headers["PRIVATE-TOKEN"], True
        except gitlab.exceptions.HttpError as he:
            if str(he) == "401 Unauthorized":
                return None, False
            else:
                raise ChisubmitException("Unexpected error getting authorization token (Reason: %s)" % (he), he)
    
    def connect(self, credentials):
        # Credentials are a GitLab private token
        self.gitlab = Gitlab(self.gitlab_hostname, token=credentials, verify_ssl=self.ssl_verify)    
        try:
            # Test connection by grabbing current user
            user = self.gitlab.currentuser()
            
            if "message" in user and user["message"] == "401 Unauthorized":
                raise ChisubmitException("Invalid GitLab credentials for server '%s'" % (self.gitlab_hostname))
            
            if "username" not in user:
                raise ChisubmitException("Unexpected error connecting to GitLab server '%s'" % (self.gitlab_hostname))
                            
        except Exception as e:
            raise
            raise ChisubmitException("Unexpected error connecting to GitLab server '%s': %s" % (self.gitlab_hostname, e))

    def disconnect(self, credentials):
        pass
    
    def init_course(self, course, fail_if_exists=True):
        group = self.__get_group(course)
        group_name = self.__get_group_name(course)
        
        if fail_if_exists and group is not None:
            raise ChisubmitException("Course '%s' already has a GitLab group" % group_name)
        
        if group is None:
            if self.staging:
                course_name = course.name + " - STAGING"
            else:
                course_name = course.name

            group_name = self.__get_group_name(course)
            new_group = self.gitlab.creategroup(course_name, group_name)
            if isinstance(new_group, gitlab.exceptions.HttpError):
                raise ChisubmitException("Could not create group '%s' (%s)" % (self.__get_group_name(course), str(new_group)), new_group)
            return True
        else:
            return False
                
    def deinit_course(self, course):
        group = self.__get_group(course)
        if group is not None:
            rv = self.gitlab.deletegroup(group["id"])
            
    def exists_user(self, course, course_user):
        gitlab_username = self._get_user_git_username(course, course_user)
        user = self.__get_user_by_username(gitlab_username)
        
        if user is None:
            return False
        else:
            return True


    def create_user(self, course, course_user):
        if self.ldap_uid_template is None:
            raise ChisubmitException("ldap_uid_template has not been set for this course")
        
        if "USER" not in self.ldap_uid_template:
            raise ChisubmitException("ldap_uid_template does not include USER: %s" % self.ldap_uid_template)
        
        gitlab_user_username = self._get_user_git_username(course, course_user)
        
        gitlab_user_name = "%s %s" % (course_user.user.first_name, course_user.user.last_name)

        gitlab_user_email = course_user.user.email
        
        # Password doesn't actually matter since we use
        # LDAP authentication. Just in case, we set it to
        # something complicated
        gitlab_user_password = ''.join([random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for i in range(25)])
        
        gitlab_extern_uid = self.ldap_uid_template.replace("USER", gitlab_user_username)
    
        self.gitlab.createuser(name = gitlab_user_name, 
                               username = gitlab_user_username, 
                               password = gitlab_user_password,
                               email = gitlab_user_email,
                               provider = "ldapmain",    # TODO: Make this configurable
                               confirm = False,
                               extern_uid = gitlab_extern_uid                           
                               )
        
    
    def update_instructors(self, course):
        instructors = course.get_instructors()
        
        usernames = [self._get_user_git_username(course, instructor) for instructor in instructors]
        
        self.__add_users_to_course_group(course, usernames, "owner")

        # TODO: Remove instructors that may have been removed


    def update_graders(self, course):
        graders = course.get_graders()
        
        usernames = [self._get_user_git_username(course, grader) for grader in graders]
        
        self.__add_users_to_course_group(course, usernames, "developer")

        # TODO: Remove instructors that may have been removed

    
    def create_team_repository(self, course, team, fail_if_exists=True, private=True):
        repo_name = self.__get_team_namespaced_project_name(course, team)
        team_members = team.get_team_members()

        student_names = ", ".join(["%s %s" % (tm.student.user.first_name, tm.student.user.last_name) for tm in team_members])
        repo_description = "%s: Team %s (%s)" % (course.name, team.team_id, student_names)
        
        if not self.staging:
            gitlab_students = []

            # Make sure users exist
            for tm in team_members:
                gitlab_student = self.__get_user_by_username(self._get_user_git_username(course, tm.student))
                if gitlab_student is None:
                    raise ChisubmitException("GitLab user '%s' does not exist " % (self._get_user_git_username(course, tm.student)))
                
                gitlab_students.append(gitlab_student)        

        project = self.__get_team_project(course, team)
        if project is not None and fail_if_exists:
            raise ChisubmitException("Repository %s already exists" % repo_name)
        
        if project is None:
            group = self.__get_group(course)
            
            if group is None:
                raise ChisubmitException("Group for course '%s' does not exist" % course.id)

            # Workaround: Our GitLab server doesn't like public repositories
            #if private:
            #    public = 0
            #else:
            #    public = 1
            
            gitlab_project = self.gitlab.createproject(team.team_id,
                                                       namespace_id = group["id"],
                                                       description = repo_description,
                                                       public = 0)
            
            if gitlab_project == False:
                raise ChisubmitException("Could not create repository %s" % repo_name)
            
            if not self.staging:                
                for gitlab_student in gitlab_students:
                    rc = self.gitlab.addprojectmember(gitlab_project["id"],
                                                      gitlab_student["id"], 
                                                      "developer")
                    
                    if rc == False:
                        raise ChisubmitException("Unable to add user %s to %s" % (gitlab_student["username"], repo_name))

                                                    
    
    def update_team_repository(self, course, team):
        repo_name = self.__get_team_namespaced_project_name(course, team)

        team_members = team.get_team_members()
        
        gitlab_project = self.__get_team_project(course, team)
        
        for tm in team_members:
            gitlab_student = self.__get_user_by_username(self._get_user_git_username(course, tm.student))
            if gitlab_student is None:
                raise ChisubmitException("GitLab user '%s' does not exist " % (self._get_user_git_username(course, tm.student)))
            rc = self.gitlab.addprojectmember(gitlab_project["id"],
                                              gitlab_student["id"], 
                                              "developer")        
            if rc == False:
                raise ChisubmitException("Unable to add user %s to %s" % (gitlab_student["username"], repo_name))
    
    
    def exists_team_repository(self, course, team):
        repo = self.__get_team_project(course, team)
        if repo is None:
            return False
        else:
            return True
    
    def get_repository_git_url(self, course, team):
        repo_name = self.__get_team_namespaced_project_name(course, team)
        hostname = self.gitlab_hostname.replace("http://","").replace("https://","")
        return "git@%s:%s.git" % (hostname, repo_name)
            
    def get_repository_http_url(self, course, team):
        repo_name = self.__get_team_namespaced_project_name(course, team)
        hostname = self.gitlab_hostname.replace("http://","").replace("https://","")
        return "https://%s/%s" % (hostname, repo_name)
    
    def get_commit(self, course, team, commit_sha):
        project_api_id = self.__get_team_project_api_id(course, team)
        gitlab_commit = self.gitlab.getrepositorycommit(project_api_id, commit_sha)
        if gitlab_commit == False:
            return None
        else:
            committer_name = gitlab_commit.get("committer_name", gitlab_commit["author_name"])
            committer_email = gitlab_commit.get("committer_email", gitlab_commit["author_email"])
            
            commit = GitCommit(gitlab_commit["id"], gitlab_commit["title"], 
                 gitlab_commit["author_name"], gitlab_commit["author_email"], parse(gitlab_commit["authored_date"]),
                 committer_name, committer_email, parse(gitlab_commit["committed_date"]))
            return commit
    
    def get_latest_commit(self, course, team, branch="master"):  
        return self.get_commit(course, team, branch)  
    
    def create_submission_tag(self, course, team, tag_name, tag_message, commit_sha):
        pass 
        # TODO: Commenting out for now, since GitLab doesn't support updating/removing
        #       tags through the API
        #        
        # project_name = self.__get_team_namespaced_project_name(course, team)
        # 
        # commit = self.get_commit(course, team, commit_sha)
        # 
        # if commit is None:
        #     raise ChisubmitException("Cannot create tag %s for commit %s (commit does not exist)" % (tag_name, commit_sha))
        # 
        # rc = self.gitlab.createrepositorytag(project_name, tag_name, commit_sha, tag_message)
        # if rc == False:
        #     raise ChisubmitException("Cannot create tag %s in project %s (error when creating tag)" % (tag_name, project_name))
    
    def update_submission_tag(self, course, team, tag_name, tag_message, commit_sha):
        # TODO: Not currently possible with current GitLab API
        pass
    
    def get_submission_tag(self, course, team, tag_name):
        project_name = self.__get_team_namespaced_project_name(course, team)
        gitlab_tag = self.__get_tag(project_name, tag_name)
        
        if gitlab_tag is None:
            return None
        
        tag = GitTag(name = gitlab_tag["name"],
             commit = self.get_commit(course, team, gitlab_tag["commit"]["id"]))
        
        return tag
    
    def delete_team_repository(self, course, team, fail_if_not_exists):
        project_name = self.__get_team_namespaced_project_name(course, team)
        project_api_id = self.__get_team_project_api_id(course, team)
        
        repo = self.__get_team_project(course, team)
        
        if repo is None:
            if fail_if_not_exists:
                raise ChisubmitException("Trying to delete a repository that doesn't exist (%s)" % (project_name))
            else:
                return 
        
        self.gitlab.deleteproject(project_api_id)
    
    def __get_group_name(self, course):
        if self.staging:
            return course.course_id + "-staging"
        else:
            return course.course_id

    def __get_user_by_username(self, username):
        # TODO: Paginations
        users = self.gitlab.getusers(search=username)
        
        if users == False:
            raise ChisubmitException("Unable to fetch Gitlab users")

        if len(users) == 0:
            return None

        for user in users:
            if user["username"] == username:
                return user

        return None    

    def __get_group(self, course):
        group = self.gitlab.getgroups(group_id = self.__get_group_name(course))

        if group == False:
            return None
        else:
            return group
        
    def __get_team_namespaced_project_name(self, course, team):
        group_name = self.__get_group_name(course)
        s = "%s/%s" % (group_name, team.team_id)
        return s.lower()      
    
    def __get_team_project_api_id(self, course, team):
        project_name = self.__get_team_namespaced_project_name(course, team)
        return project_name.replace("/", "%2F")
    
    def __get_team_project(self, course, team):
        namespaced_project_name = self.__get_team_namespaced_project_name(course, team)
        project = self.gitlab.getproject(namespaced_project_name)
        
        if project == False:
            return None
        else:
            return project      
        
    def __add_users_to_course_group(self, course, usernames, access_level):
        group_name = self.__get_group_name(course)
        group = self.__get_group(course)

        if group is None:
            raise ChisubmitException("Couldn't add users '%s' to group '%s'. Course group does not exist" % (usernames, group_name))

        users = []
        for username in usernames: 
            user = self.__get_user_by_username(username)
            if user is None:
                raise ChisubmitException("Couldn't add user '%s' to group '%s'. User does not exist" % (username, group_name))
            users.append(user)
        
        for user in users:    
            self.gitlab.addgroupmember(group["id"], user["id"], access_level)
        
        # If the return code is False, we can't distinguish between
        # "failed because the user is already in the group" or
        # "failed for other reason".
        
        # TODO: Check whether user was actually added to group
    
    def __get_tag(self, project_name, tag_name):
        tags = self.gitlab.getrepositorytags(project_name)
        
        if tags == False:
            raise ChisubmitException("Couldn't get tags for project %s" % project_name)
        
        for t in tags:
            if t["id"] == tag_name:
                return t
            
        return None
            
        
    def __has_tag(self, project_name, tag_name):
        return self.__get_tag(project_name, tag_name) is not None            
Exemple #55
-1
def get_projects(config):
    url = config.get('gitlab', 'url')
    gl = Gitlab(url, get_token(config))
    logger.debug("Connecting to Gitlab {}".format(url))
    gl.auth() # XXX catch exceptions
    user = gl.user
    name = user.name.encode('utf8')
    username = user.username.encode('utf8')
    logger.info("Connected as {1} ({0})".format(
        user.name.encode('utf8'), user.username))
    group = gl.Group(config.get('gitlab', 'group')) # XXX catch exceptions
    return group.projects