def test_authentication_error_is_raised_with_invalid_private_token(self):
"""
Assert that a `401 Unauthorized` exception is raised provided the user
enters invalid credentials
"""
with self.assertRaises(GitLabServerError) as e:
gitlab = GitLab(host='gitlab.com', private_token='foobar')
gitlab.get_current_user()
exception = e.exception
self.assertEqual(exception.status_code, 401)
self.assertEqual(exception.reason, 'Unauthorized')
def updatePullRequests(self):
print 'Updating pull requests from GitLab...'
if not self.client:
self.client = GitLab("https://gitlab.itseez.com/api/v3", userAgent=userAgent, private_token=self.gitlab_private_token, async=True)
pullrequests = yield self.client.projects(self.project_id).merge_requests.get(state='opened', per_page=100)
if self.client.status == 304:
print "GitLab merge requests are not changed"
defer.returnValue(None)
elif self.client.status == 200:
projects_info = {}
prs = []
for pullrequest in pullrequests:
try:
newAPI = False
pr = {}
if not pullrequest['state'] in ['opened', 'reopened']:
continue
pr['id'] = pullrequest['iid']
pr['branch'] = pullrequest['target_branch']
pr['author'] = pullrequest['author']['username']
pr['assignee'] = pullrequest['assignee']['username'] if pullrequest['assignee'] else None
if newAPI:
if not projects_info.has_key(pullrequest['source_project_id']):
projects_info[pullrequest['source_project_id']] = yield self.client.projects(pullrequest['source_project_id']).get()
pr['head_user'] = projects_info[pullrequest['source_project_id']]['owner']['username']
pr['head_repo'] = projects_info[pullrequest['source_project_id']]['path_with_namespace']
else: # Old API
pr['head_user'] = self.username
pr['head_repo'] = '%s/%s' % (self.username, self.repo)
pr['head_branch'] = pullrequest['source_branch']
if newAPI:
branch_info = yield self.client.projects(pullrequest['source_project_id']).repository.branches(pullrequest['source_branch']).get()
else:
branch_info = yield self.client.projects(pullrequest['project_id']).repository.branches(pullrequest['source_branch']).get()
pr['head_sha'] = branch_info['commit']['id']
pr['title'] = pullrequest['title']
pr['description'] = pullrequest.get('description', pullrequest['title'])
if pr['description'] is None:
pr['description'] = ''
prs.append(pr)
except:
f = failure.Failure()
log.err(f, 'while adding merge request')
pass
defer.returnValue(prs)
raise Exception('invalid status', self.client.status)
class GitLabContext(pullrequest.context.Context):
updatePullRequestsDelay = 30
name = 'GitLab Pull Requests'
dbname = 'pullrequests_gitlab'
urlpath = 'pullrequests_gl'
builders = dict(
linux=dict(name='Linux x64', builders=['precommit_linux64'], order=100),
windows=dict(name='Win x64', builders=['precommit_windows64'], order=200),
win32=dict(name='Win 32', builders=['precommit_windows32'], order=250),
macosx=dict(name='Mac', builders=['precommit_macosx'], order=300),
android=dict(name='Android', builders=['precommit_android'], order=400),
)
username = '' # TODO
repo = '' # TODO
project_id = -1 # curl --header "PRIVATE-TOKEN: xXxXxXxXxXx" "https://gitlab.itseez.com/api/v3/projects/"
gitlab_private_token = os.environ['GITLAB_APIKEY'] # check deploy/apikeys.sh
client = None
@defer.inlineCallbacks
def updatePullRequests(self):
print 'Updating pull requests from GitLab...'
if not self.client:
self.client = GitLab("https://gitlab.itseez.com/api/v3", userAgent=userAgent, private_token=self.gitlab_private_token, async=True)
pullrequests = yield self.client.projects(self.project_id).merge_requests.get(state='opened', per_page=100)
if self.client.status == 304:
print "GitLab merge requests are not changed"
defer.returnValue(None)
elif self.client.status == 200:
projects_info = {}
prs = []
for pullrequest in pullrequests:
try:
newAPI = False
pr = {}
if not pullrequest['state'] in ['opened', 'reopened']:
continue
pr['id'] = pullrequest['iid']
pr['branch'] = pullrequest['target_branch']
pr['author'] = pullrequest['author']['username']
pr['assignee'] = pullrequest['assignee']['username'] if pullrequest['assignee'] else None
if newAPI:
if not projects_info.has_key(pullrequest['source_project_id']):
projects_info[pullrequest['source_project_id']] = yield self.client.projects(pullrequest['source_project_id']).get()
pr['head_user'] = projects_info[pullrequest['source_project_id']]['owner']['username']
pr['head_repo'] = projects_info[pullrequest['source_project_id']]['path_with_namespace']
else: # Old API
pr['head_user'] = self.username
pr['head_repo'] = '%s/%s' % (self.username, self.repo)
pr['head_branch'] = pullrequest['source_branch']
if newAPI:
branch_info = yield self.client.projects(pullrequest['source_project_id']).repository.branches(pullrequest['source_branch']).get()
else:
branch_info = yield self.client.projects(pullrequest['project_id']).repository.branches(pullrequest['source_branch']).get()
pr['head_sha'] = branch_info['commit']['id']
pr['title'] = pullrequest['title']
pr['description'] = pullrequest.get('description', pullrequest['title'])
if pr['description'] is None:
pr['description'] = ''
prs.append(pr)
except:
f = failure.Failure()
log.err(f, 'while adding merge request')
pass
defer.returnValue(prs)
raise Exception('invalid status', self.client.status)
def getListOfAutomaticBuilders(self, pr):
if pr.description is not None and '**WIP**' in pr.description:
return []
buildersList = ['linux', 'windows', 'win32', 'macosx', 'android']
return buildersList
def getBuildProperties(self, pr, b, properties, sourcestamps):
prid = pr.prid
properties.setProperty('branch', pr.branch, 'Pull request')
properties.setProperty('head_sha', pr.head_sha, 'Pull request')
properties.setProperty('pullrequest', prid, 'Pull request')
if b.isPerf:
regressionTestFilter = self.extractRegressionTestFilter(pr.description)
if regressionTestFilter is not None:
properties.setProperty('regression_test_filter', regressionTestFilter, 'Pull request')
else:
print 'ERROR: Can\'t schedule perf precommit build without regression test filter. Use check_regression parameter'
defer.returnValue(False)
if pr.description is None or '**WIP**' in pr.description:
self.pushBuildProperty(properties, pr.description, 'test[s]?_filter[s]?', 'test_filter')
sourcestamps.append(dict(
codebase='code',
repository='ssh://[email protected]/xxx/yyy.git',
branch=pr.branch))
sourcestamps.append(dict(
codebase='code_merge',
repository='ssh://[email protected]/xxx/yyy.git',
branch=pr.head_branch,
revision=pr.head_sha))
return True
def getWebAddressPullRequest(self, pr):
return 'https://gitlab.itseez.com/xxx/yyy/merge_requests/%d' % (pr.prid)
def getWebAddressPerfRegressionReport(self, pr):
return None
def main(argv=None):
'''
Process the command line arguments and create the JSON dump.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Export all users/issues from GitLab to JIRA JSON format.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('gitlab_url',
help='The full URL to your GitLab instance.')
parser.add_argument('-d',
'--date_filter',
help='Only include issues, notes, etc. created after\
the specified date. Expected format is \
YYYY-MM-DD',
type=get_datetime,
default='1970-01-01')
parser.add_argument('-e',
'--include_empty',
help='Include projects in output that do not have any\
issues.',
action='store_true')
parser.add_argument('-i',
'--ignore_list',
help='List of project names to exclude from dump.',
type=argparse.FileType('r'))
parser.add_argument('-p',
'--password',
help='The password to use to authenticate if token is \
not specified. If password and token are both \
unspecified, you will be prompted to enter a \
password.')
parser.add_argument('-P',
'--page_size',
help='When retrieving result from GitLab, how many \
results should be included in a given page?.',
type=int,
default=20)
parser.add_argument('-s',
'--verify_ssl',
help='Enable SSL certificate verification',
action='store_true')
parser.add_argument('-t',
'--token',
help='The private GitLab API token to use for \
authentication. Either this or username and \
password must be set.')
parser.add_argument('-u',
'--username',
help='The username to use for authentication, if token\
is unspecified.')
parser.add_argument('-v',
'--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0,
action='count')
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
args.page_size = max(100, args.page_size)
# Convert verbose flag to actually logging level
log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
log_level = log_levels[min(args.verbose, 2)]
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'),
level=log_level)
# Setup authenticated GitLab instance
if args.token:
git = GitLab(args.gitlab_url,
token=args.token,
verify_ssl=args.verify_ssl)
else:
if not args.username:
print('Username: '******'').strip()
if not args.password:
args.password = getpass.getpass('Password: '******'Creating project entries...', end="", file=sys.stderr)
sys.stderr.flush()
key_set = set()
mentioned_users = set()
if args.ignore_list is not None:
ignore_list = {line.strip().lower() for line in args.ignore_list}
else:
ignore_list = {}
for project in gen_all_results(git.getallprojects,
per_page=args.page_size):
proj_name_lower = project['name'].lower()
if proj_name_lower not in ignore_list and project['issues_enabled']:
project_issues = []
for issue in gen_all_results(git.getprojectissues,
project['id'],
per_page=args.page_size):
if args.date_filter < datetime.strptime(
issue['updated_at'], TIME_FORMAT):
project_issues.append(issue)
else:
for note in git.getissuewallnotes(project['id'],
issue['id']):
if (args.date_filter < datetime.strptime(
note['created_at'], TIME_FORMAT)):
project_issues.append(issue)
break
if project_issues or args.include_empty:
jira_project = {}
jira_project['name'] = project['name_with_namespace']
key = project['name']
if key.islower():
key = key.title()
key = re.sub(r'[^A-Z]', '', key)
if len(key) < 2:
key = re.sub(r'[^A-Za-z]', '',
project['name'])[0:2].upper()
added = False
suffix = 65
while key in key_set:
if not added:
key += 'A'
else:
suffix += 1
key = key[:-1] + chr(suffix)
key_set.add(key)
jira_project['key'] = key
jira_project['description'] = md_to_wiki(
project['description'])
# jira_project['created'] = project['created_at']
jira_project['issues'] = []
for issue in project_issues:
jira_issue = {}
jira_issue['externalId'] = issue['iid']
if issue['state'] == 'closed':
jira_issue['status'] = 'Closed'
jira_issue['resolution'] = 'Resolved'
else:
jira_issue['status'] = 'Open'
jira_issue['description'] = md_to_wiki(
issue['description'])
jira_issue['reporter'] = issue['author']['username']
mentioned_users.add(jira_issue['reporter'])
jira_issue['labels'] = issue['labels']
jira_issue['summary'] = issue['title']
if issue['assignee']:
jira_issue['assignee'] = issue['assignee']['username']
mentioned_users.add(jira_issue['assignee'])
jira_issue['issueType'] = 'Bug'
jira_issue['comments'] = []
# Get all comments/notes
for note in git.getissuewallnotes(project['id'],
issue['id']):
jira_note = {}
jira_note['body'] = md_to_wiki(note['body'])
jira_note['author'] = note['author']['username']
mentioned_users.add(jira_note['author'])
jira_note['created'] = note['created_at']
jira_issue['comments'].append(jira_note)
jira_project['issues'].append(jira_issue)
output_dict['projects'].append(jira_project)
print('.', end="", file=sys.stderr)
sys.stderr.flush()
print('\nCreating user entries...', end="", file=sys.stderr)
sys.stderr.flush()
for user in gen_all_results(git.getusers, per_page=args.page_size):
# Only add users who are actually referenced in issues
if user['username'] in mentioned_users:
jira_user = {}
jira_user['name'] = user['username']
jira_user['fullname'] = user['name']
jira_user['email'] = user['email']
jira_user['groups'] = ['gitlab-users']
jira_user['active'] = (user['state'] == 'active')
output_dict['users'].append(jira_user)
print('.', end="", file=sys.stderr)
sys.stderr.flush()
print('\nPrinting JSON output...', file=sys.stderr)
sys.stderr.flush()
print(json.dumps(output_dict, indent=4))
def main(argv=None):
'''
Process the command line arguments and create the JSON dump.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv[1:]`` is used instead.
:type argv: list of str
'''
# Get command line arguments
parser = argparse.ArgumentParser(
description="Transfer all projects/repositories from GitLab to Stash. \
Note: This script assumes you have your SSH key \
registered with both GitLab and Stash.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve')
parser.add_argument('gitlab_url',
help='The full URL to your GitLab instance.')
parser.add_argument('stash_url',
help='The full URL to your Stash instance.')
parser.add_argument('-p',
'--password',
help='The password to use to authenticate if token is \
not specified. If password and token are both \
unspecified, you will be prompted to enter a \
password.')
parser.add_argument('-P',
'--page_size',
help='When retrieving result from GitLab, how many \
results should be included in a given page?.',
type=int,
default=20)
parser.add_argument('-s',
'--verify_ssl',
help='Enable SSL certificate verification',
action='store_true')
parser.add_argument('-S',
'--skip_existing',
help='Do not update existing repositories and just \
skip them.',
action='store_true')
parser.add_argument('-t',
'--token',
help='The private GitLab API token to use for \
authentication. Either this or username and \
password must be set.')
parser.add_argument('-u',
'--username',
help='The username to use for authentication, if token\
is unspecified.')
parser.add_argument('-v',
'--verbose',
help='Print more status information. For every ' +
'additional time this flag is specified, ' +
'output gets more verbose.',
default=0,
action='count')
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
args.page_size = max(100, args.page_size)
# Convert verbose flag to actually logging level
log_levels = [logging.WARNING, logging.INFO, logging.DEBUG]
log_level = log_levels[min(args.verbose, 2)]
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'),
level=log_level)
# Setup authenticated GitLab and Stash instances
if args.token:
git = GitLab(args.gitlab_url,
token=args.token,
verify_ssl=args.verify_ssl)
else:
git = None
if not args.username:
print('Username: '******'').strip()
if not args.password:
args.password = getpass.getpass('Password: '******'Retrieving existing Stash projects...', end="", file=sys.stderr)
sys.stderr.flush()
key_set = {proj['key'] for proj in stash.projects}
stash_project_names = {proj['name'] for proj in stash.projects}
names_to_keys = {proj['name']: proj['key'] for proj in stash.projects}
print('done', file=sys.stderr)
sys.stderr.flush()
updated_projects = set()
repo_to_slugs = {}
failed_to_clone = set()
cwd = os.getcwd()
transfer_count = 0
skipped_count = 0
print('Processing GitLab projects...', file=sys.stderr)
sys.stderr.flush()
for project in gen_all_results(git.getallprojects,
per_page=args.page_size):
print('\n' + ('=' * 80) + '\n', file=sys.stderr)
sys.stderr.flush()
proj_name = project['namespace']['name']
# Create Stash project if it doesn't already exist
if proj_name not in stash_project_names:
# Create Stash project key
key = proj_name
if key.islower():
key = key.title()
key = re.sub(r'[^A-Z]', '', key)
if len(key) < 2:
key = re.sub(r'[^A-Za-z]', '', proj_name)[0:2].upper()
added = False
suffix = 65
while key in key_set:
if not added:
key += 'A'
else:
suffix += 1
key = key[:-1] + chr(suffix)
key_set.add(key)
# Actually add the project to Stash
print('Creating Stash project "%s" with key %s...' %
(proj_name, key),
end="",
file=sys.stderr)
sys.stderr.flush()
stash.projects.create(key, proj_name)
names_to_keys[proj_name] = key
stash_project_names.add(proj_name)
print('done', file=sys.stderr)
sys.stderr.flush()
else:
key = names_to_keys[proj_name]
stash_project = stash.projects[key]
# Initialize maping from repository names to slugs for later
if key not in repo_to_slugs:
repo_to_slugs[key] = {
repo['name']: repo['slug']
for repo in stash_project.repos
}
# Create Stash-compatible name for repository
# Repository names are limited to 128 characters.
# They must start with a letter or number and may contain spaces,
# hyphens, underscores and periods
repo_name = project['name']
if not repo_name[0].isalnum():
repo_name = 'A ' + repo_name
repo_name = re.sub(r'[^A-Za-z0-9 _.-]', ' ', repo_name)
if len(repo_name) > 128:
repo_name = repo_name[0:128]
# Add repository to Stash project if it's not already there
if repo_name not in repo_to_slugs[key]:
print('Creating Stash repository "%s" in project "%s"...' %
(repo_name, proj_name),
end="",
file=sys.stderr)
sys.stderr.flush()
stash_repo = stash_project.repos.create(repo_name)
repo_to_slugs[key][repo_name] = stash_repo['slug']
print('done', file=sys.stderr)
sys.stderr.flush()
elif args.skip_existing:
print('Skipping existing Stash repository "%s" in project "%s"' %
(repo_name, proj_name),
file=sys.stderr)
sys.stderr.flush()
skipped_count += 1
continue
else:
print('Updating existing Stash repository "%s" in project "%s"' %
(repo_name, proj_name),
file=sys.stderr)
sys.stderr.flush()
repo_slug = repo_to_slugs[key][repo_name]
stash_repo = stash_project.repos[repo_slug].get()
for clone_link in stash_repo['links']['clone']:
if clone_link['name'] == 'ssh':
stash_repo_url = clone_link['href']
break
with tempfile.TemporaryDirectory() as temp_dir:
# Clone repository to temporary directory
print('\nCloning GitLab repository...', file=sys.stderr)
sys.stderr.flush()
try:
subprocess.check_call([
'git', 'clone', '--mirror', project['ssh_url_to_repo'],
temp_dir
])
except subprocess.CalledProcessError:
print('Failed to clone GitLab repository. This usually when ' +
'it does not exist.',
file=sys.stderr)
failed_to_clone.add(project['name_with_namespace'])
skipped_count += 1
continue
os.chdir(temp_dir)
# Check that repository is not empty
try:
subprocess.check_call(['git', 'log', '--format=oneline', '-1'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError:
print('Repository is empty, so skipping push to Stash.',
file=sys.stderr)
skipped_count += 1
else:
# Change remote to Stash and push
print('\nPushing repository to Stash...', file=sys.stderr)
sys.stderr.flush()
subprocess.check_call(
['git', 'remote', 'set-url', 'origin', stash_repo_url])
subprocess.check_call(['git', 'push', '--mirror'])
transfer_count += 1
os.chdir(cwd)
updated_projects.add(proj_name)
print('\n' + ('=' * 35) + 'SUMMARY' + ('=' * 35), file=sys.stderr)
print('{} repositories transferred.\n'.format(transfer_count),
file=sys.stderr)
print('{} repositories skipped.\n'.format(skipped_count), file=sys.stderr)
print('Projects created/updated:', file=sys.stderr)
for proj in sorted(updated_projects):
print('\t' + proj, file=sys.stderr)
print('Repositories that we could not clone:', file=sys.stderr)
for repo_name in sorted(failed_to_clone):
print('\t' + repo_name, file=sys.stderr)
class GitLabTestCase(unittest.TestCase):
def setUp(self):
self.config = ConfigParser()
self.config.read('~/.gitlab')
self.gitlab = GitLab(host='gitlab.com')
def test_authentication_error_is_raised_with_invalid_private_token(self):
"""
Assert that a `401 Unauthorized` exception is raised provided the user
enters invalid credentials
"""
with self.assertRaises(GitLabServerError) as e:
gitlab = GitLab(host='gitlab.com', private_token='foobar')
gitlab.get_current_user()
exception = e.exception
self.assertEqual(exception.status_code, 401)
self.assertEqual(exception.reason, 'Unauthorized')
def test_private_token_http_header_exists(self):
"""
Assert that the `PRIVATE-TOKEN` header exists in the HTTP request
header and that it is not `None`
"""
self.assertTrue('PRIVATE-TOKEN' in self.gitlab._session.headers)
self.assertIsNotNone(self.gitlab._session.headers['PRIVATE-TOKEN'])
def test_get_users_endpoint_with_query_params(self):
"""
Assert that the result set returned with query params is as expected
"""
users = self.gitlab.get_users(per_page=1)
self.assertEqual(len(users), 1)
def test_get_user_endpoint_with_invalid_id(self):
"""
Assert that `400 Bad Request` is raised when retrieving a user with
invalid ID
"""
with self.assertRaises(GitLabServerError) as e:
self.gitlab.get_user(id=-1)
exception = e.exception
self.assertEqual(exception.status_code, 400)
self.assertEqual(exception.reason, 'Bad Request')
def test_get_projects_endpoint_with_query_params(self):
"""
Assert that the result set returned with query params is as expected
"""
projects = self.gitlab.get_projects(sort='asc')
format_spec = '%Y-%m-%dT%H:%M:%S.%fZ'
# Convert ISO formatted `datetime` strings into `datetime` objects
projects = [
datetime.strptime(project['created_at'], format_spec)
for project in projects
]
# Need to create a copy to not modify the original `projects` list
# object
projects_copy = list(projects)
# Assert that the `projects` are sorted in ascending order
self.assertEqual(projects, sorted(projects_copy))
projects = self.gitlab.get_projects(sort='desc')
projects = [
datetime.strptime(project['created_at'], format_spec)
for project in projects
]
# `reversed` function returns a generator object, need to convert to a
# `list` object
self.assertEqual(projects, list(reversed(projects_copy)))
def test_namespace_decorator(self):
"""
Assert that project namespaces are URL encoded
(e.g. / is represented by %2F)
"""
@namespace
def foo(id=None):
self.assertTrue('%2F' in id)
foo.__call__(id='foo/bar')
def setUp(self):
self.config = ConfigParser()
self.config.read('~/.gitlab')
self.gitlab = GitLab(host='gitlab.com')
import os
from gitlab import GitLab
client = GitLab("https://gitlab.itseez.com/api/v3", "Test", private_token=os.environ['GITLAB_APIKEY'])
res = client.projects().get()
print res
