def test_get_last_date(self):
"""Test whether the last date is correctly returned"""
items = json.loads(read_file('data/git.json'))
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
new_items = elastic.bulk_upload(items, field_id="uuid")
self.assertEqual(new_items, 11)
# no filter
last_date = elastic.get_last_date('updated_on')
self.assertEqual(last_date.isoformat(), '2019-10-01T18:05:52+00:00')
# filter including all items
fltr = {
'name': 'origin',
'value': '/tmp/perceval_mc84igfc/gittest'
}
last_date = elastic.get_last_date('updated_on', filters_=[fltr])
self.assertEqual(last_date.isoformat(), '2014-02-12T06:11:12+00:00')
# filter including a sub-set og items
fltr = {
'name': 'perceval_version',
'value': '0.9.11'
}
last_date = elastic.get_last_date('updated_on', filters_=[fltr])
self.assertEqual(last_date.isoformat(), '2014-02-12T06:09:04+00:00')
try:
repos = get_repositores(owner_url, args.token, args.nrepos)
except requests.exceptions.HTTPError:
logging.error("Can't get repos for %s" % (owner_url))
continue
if args.db_projects_map:
insert_projects_mapping(args.db_projects_map, org, repos)
for repo in repos:
repo_url = repo['clone_url']
origin = repo_url
clone_dir = path.join(GIT_CLONE_DIR, repo_url.replace("/", "_"))
filter_ = {"name": "origin", "value": origin}
last_update = None
if es_enrich:
last_update = es_enrich.get_last_date("metadata__updated_on", filter_)
if last_update:
last_update = last_update.isoformat()
repo_args = {
"gitpath": clone_dir,
"uri": repo_url,
"cache": False
}
if last_update:
repo_args["from_date"] = last_update
arthur_repos["repositories"].append({
"args": repo_args,
"backend": PERCEVAL_BACKEND,
"origin": repo_url,
"elastic_index": PERCEVAL_BACKEND
})
