def fetch_gitlab():
"""
Fetches the data items(issues and merge requests)
from a GitLab repository
"""
gitlab = GitLab(owner=owner, repository=repository, api_token=args.token, sleep_for_rate=True)
# Printing Owner and Repository
print("Owner: ", owner)
print("Repository: ", repository)
print("Categories: ", GitLab.CATEGORIES)
# Date from which data is to be fetched
from_date = datetime.datetime(2020, 3, 10, 0, 0, 0, tzinfo=dateutil.tz.tzutc())
# Fetch Issue data
issue_list_generator = gitlab.fetch(category=CATEGORY_ISSUE, from_date=from_date)
issue_list = list(issue_list_generator)
# Dump the data fetched into a JSON file
dump_json("./GitLab_backend/gitlab_issue.json", issue_list)
issue = issue_list[0]
# Printing some features of the issue
print('*' * 50)
print("ISSUE")
print('Category: ', issue['category'])
print("Issue Count: ", len(issue_list))
print('Title: ', issue['data']['title'])
print('Notes Count: ', len(issue['data']['notes_data']))
print('Search Fields:', issue['search_fields'])
print('Timestamp: ', issue['timestamp'])
print('Updated on: ', issue['updated_on'])
print('UUID: ', issue['uuid'])
print('*' * 50)
# Fetch Merge Request data
pr_list_generator = gitlab.fetch(category=CATEGORY_MERGE_REQUEST, from_date=from_date)
pr_list = list(pr_list_generator)
# Dump the data fetched into a JSON file
dump_json("./GitLab_backend/gitlab_mr.json", pr_list)
pr = pr_list[0]
# Printing some features of the repository data
print("PULL REQUEST")
print('Category: ', pr['category'])
print("Merge Request Count: ", len(pr_list))
print('Title: ', pr['data']['title'])
print('Notes Count: ', len(pr['data']['notes_data']))
print('Search Fields:', pr['search_fields'])
print('Timestamp: ', pr['timestamp'])
print('Updated on: ', pr['updated_on'])
print('UUID: ', pr['uuid'])
print('*' * 50)
def test_fetch_from_date(self):
"""Test whether issues from a given date are properly fetched from GitLab"""
setup_http_server(GITLAB_URL_PROJECT, GITLAB_ISSUES_URL)
gitlab = GitLab("fdroid", "fdroiddata", "your-token")
from_date = datetime.datetime(2017, 3, 18)
issues = [issues for issues in gitlab.fetch(from_date=from_date)]
self.assertEqual(len(issues), 3)
self.assertEqual(issues[0]['origin'], GITLAB_URL + '/fdroid/fdroiddata')
self.assertEqual(issues[0]['category'], 'issue')
self.assertEqual(issues[0]['tag'], GITLAB_URL + '/fdroid/fdroiddata')
self.assertEqual(issues[0]['data']['author']['id'], 1)
self.assertEqual(issues[0]['data']['author']['username'], 'redfish64')
from_date = datetime.datetime(3019, 3, 18)
issues = [issues for issues in gitlab.fetch(from_date=from_date)]
self.assertEqual(len(issues), 0)
def extract_gitlab(own, repo):
# Initialising GitLab object
gitlab_obj = GitLab(owner=own,
repository=repo,
sleep_for_rate=True,
sleep_time=300,
api_token='xxxx')
"""The method fetch() retrieves a list of all issues/merge requests
from a GitLab repository.
Here, we are fetching the list of commits and printing all the data.
"""
for commit in gitlab_obj.fetch():
print(commit['data'])
def test_fetch_enterprise(self):
"""Test whether issues are properly fetched from GitLab Enterprise server"""
setup_http_server(GITLAB_ENTERPRISE_URL_PROJECT, GITLAB_ENTERPRISE_ISSUES_URL)
gitlab = GitLab('am', 'test', base_url=GITLAB_ENTERPRISE_URL)
issues = [issues for issues in gitlab.fetch()]
self.assertEqual(len(issues), 4)
self.assertEqual(issues[0]['origin'], GITLAB_ENTERPRISE_URL + "/am/test")
self.assertEqual(issues[0]['category'], 'issue')
self.assertEqual(issues[0]['tag'], GITLAB_ENTERPRISE_URL + "/am/test")
self.assertEqual(issues[0]['data']['author']['id'], 1)
self.assertEqual(issues[0]['data']['author']['username'], 'redfish64')
def test_fetch(self):
"""Test whether issues are properly fetched from GitLab"""
setup_http_server(GITLAB_URL_PROJECT, GITLAB_ISSUES_URL)
gitlab = GitLab("fdroid", "fdroiddata", "your-token")
issues = [issues for issues in gitlab.fetch()]
self.assertEqual(len(issues), 4)
self.assertEqual(issues[0]['origin'], GITLAB_URL + '/fdroid/fdroiddata')
self.assertEqual(issues[0]['category'], 'issue')
self.assertEqual(issues[0]['tag'], GITLAB_URL + '/fdroid/fdroiddata')
self.assertEqual(issues[0]['data']['author']['id'], 1)
self.assertEqual(issues[0]['data']['author']['username'], 'redfish64')
def test_fetch_empty(self):
"""Test when return empty"""
page_1 = ''
httpretty.register_uri(httpretty.GET,
GITLAB_URL_PROJECT,
body='',
status=200)
httpretty.register_uri(httpretty.GET,
GITLAB_ISSUES_URL,
body=page_1,
status=200)
gitlab = GitLab("fdroid", "fdroiddata", api_token="your-token")
issues = [issues for issues in gitlab.fetch()]
self.assertEqual(len(issues), 0)
parser.add_argument("-o", "--owner", help = "GitLab owner/org", required=True, type=str)
parser.add_argument("-r", "--repo", help = "GitLab repo", required=True, type=str)
parser.add_argument("-s", "--sleep", help = "Sleep for rate", type=bool, default=False)
parser.add_argument("-T", "--token", help = "GitLab token", type=str)
parser.add_argument("-c", "--category", help = "Gitlab category (issue or merge_request)", type=str, default="issue")
parser.add_argument("-C", "--use-created-date", help = "Use created date instead of update date", type=lambda s: s.lower() in ['true', 't', 'yes', 'y', '1'])
parser.add_argument("-D", "--updated-diff", help = "If >=0 skip objects where created + diff > updated", type=int, default=-1)
args = parser.parse_args()
# print(args)
gitlab = GitLab(owner=args.owner, repository=args.repo, api_token=args.token, base_url=args.url, tag=None, archive=None, sleep_for_rate=args.sleep, min_rate_to_sleep=10, max_retries=5, sleep_time=1)
# print(gitlab)
# print ((args.date_from, args.date_to))
oids = set()
for obj in gitlab.fetch(category=args.category, from_date=args.date_from):
# print(obj.keys())
# print(obj['data'].keys())
# dtu = dateutil.parser.parse(obj['data']['updated_at'])
dtu = utc.localize(datetime.datetime.fromtimestamp(gitlab.metadata_updated_on(obj['data'])))
# print(dtu)
if args.use_created_date:
dtc = dateutil.parser.parse(obj['data']['created_at'])
diff = (dtu - dtc) / datetime.timedelta(seconds=1)
# print((dtc, dtu))
if (args.updated_diff >= 0 and diff > args.updated_diff) or dtc < args.date_from or dtc > args.date_to:
# print("skip {0},{1}".format(dtc, dtu))
continue
elif dtu > args.date_to:
# print("skip {0}".format(dtu))
break