def test_search_series_tags(self):
# nosetests -s -v widukind_common.tests.test_tags:SearchTagsTestCase.test_search_series_tags
self.db[constants.COL_PROVIDERS].insert(self.doc_provider)
self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1)
self.db[constants.COL_DATASETS].insert(self.doc_dataset)
self.assertEqual(self.db[constants.COL_DATASETS].count(), 1)
tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"])
self.db[constants.COL_SERIES].insert(self.doc_series)
self.assertEqual(self.db[constants.COL_SERIES].count(), 1)
tags_utils.update_tags_series(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"])
def _update_tags(ctx, db, provider_name, dataset=None, max_bulk=100, update_only=False, dry_mode=False, async_mode=None):
start = time.time()
ctx.log("START update tags for [%s]" % provider_name)
ctx.log("Update provider[%s] Datasets tags..." % provider_name)
try:
result = tags.update_tags_datasets(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_warn("Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err)))
ctx.log("Update provider[%s] Series tags..." % provider_name)
try:
result = tags.update_tags_series(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
async_mode=async_mode,
dry_mode=dry_mode)
if not async_mode:
ctx.log_warn("Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception:
ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error()))
end = time.time() - start
ctx.log("update tags END: provider[%s] - time[%.3f]" % (provider_name, end))
def test_search_series_tags(self):
# nosetests -s -v widukind_common.tests.test_tags:SearchTagsTestCase.test_search_series_tags
self.db[constants.COL_PROVIDERS].insert(self.doc_provider)
self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1)
self.db[constants.COL_DATASETS].insert(self.doc_dataset)
self.assertEqual(self.db[constants.COL_DATASETS].count(), 1)
tags_utils.update_tags_datasets(self.db, self.doc_provider["name"],
self.doc_dataset["dataset_code"])
self.db[constants.COL_SERIES].insert(self.doc_series)
self.assertEqual(self.db[constants.COL_SERIES].count(), 1)
tags_utils.update_tags_series(self.db, self.doc_provider["name"],
self.doc_dataset["dataset_code"])
def _update_tags(ctx,
db,
provider_name,
dataset=None,
max_bulk=100,
update_only=False,
dry_mode=False,
async_mode=None):
start = time.time()
ctx.log("START update tags for [%s]" % provider_name)
ctx.log("Update provider[%s] Datasets tags..." % provider_name)
try:
result = tags.update_tags_datasets(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_warn(
"Update provider[%s] Datasets tags Success. Docs Updated[%s]" %
(provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" %
(provider_name, str(err)))
ctx.log("Update provider[%s] Series tags..." % provider_name)
try:
result = tags.update_tags_series(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
async_mode=async_mode,
dry_mode=dry_mode)
if not async_mode:
ctx.log_warn(
"Update provider[%s] Series tags Success. Docs Updated[%s]" %
(provider_name, result["nModified"]))
except Exception:
ctx.log_error("Update Series tags Fail - provider[%s]: %s" %
(provider_name, last_error()))
end = time.time() - start
ctx.log("update tags END: provider[%s] - time[%.3f]" %
(provider_name, end))
def test_update_tags_series(self):
# nosetests -s -v widukind_common.tests.test_tags:UpdateTagsTestCase.test_update_tags_series
self.db[constants.COL_PROVIDERS].insert(self.doc_provider)
self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1)
dataset = {
"enable": True,
"provider_name": self.doc_provider["name"],
"dataset_code": "d1",
"name": "dataset 1",
"slug": "%s-d1" % self.doc_provider["slug"],
"concepts": {
"FREQ": "Frequency",
"OBS_STATUS": "Observation Status",
"COUNTRY": "Country"
},
"codelists": {
"FREQ": {
"D": "Daily",
"M": "Monthly"
},
"OBS_STATUS": {
"E": "Estimate",
},
"COUNTRY": {
"FRA": "France"
}
},
"dimension_keys": ["FREQ", "COUNTRY"],
"attribute_keys": ["OBS_STATUS"],
}
self.db[constants.COL_DATASETS].insert(dataset)
self.assertEqual(self.db[constants.COL_DATASETS].count(), 1)
series = {
"provider_name": self.doc_provider["name"],
"dataset_code": dataset["dataset_code"],
"key": "x1",
"name": "series 1",
"slug": "%s-d1-x1" % self.doc_provider["slug"],
"frequency": "M",
"dimensions": {
"COUNTRY": "FRA"
},
"attributes": {
"OBS_STATUS": "E"
},
}
self.db[constants.COL_SERIES].insert(series)
self.assertEqual(self.db[constants.COL_SERIES].count(), 1)
result = tags_utils.update_tags_series(self.db,
self.doc_provider["name"],
dataset["dataset_code"])
self.assertEqual(result["nMatched"], 1)
self.assertEqual(result["nModified"], 1)
result = tags_utils.update_tags_series(self.db,
self.doc_provider["name"],
dataset["dataset_code"],
update_only=True)
self.assertEqual(result["nMatched"], 0)
self.assertEqual(result["nModified"], 0)
query = {"provider_name": series["provider_name"],
"dataset_code": series["dataset_code"],
"key": series["key"]}
series_doc = self.db[constants.COL_SERIES].find_one(query)
self.assertIsNotNone(series_doc)
self.assertEqual(series_doc["tags"], ['country', 'd1', 'dataset', 'estimate', 'france', 'mars', 'monthly', 'observation', 'p1', 'provider', 'series', 'status', 'test', 'x1'])
def cmd_update_tags(fetcher=None, dataset=None, max_bulk=100,
update_only=False, async_mode=None,
dry_mode=False, **kwargs):
"""Create or Update field tags"""
"""
Examples:
dlstats fetchers tag -f BIS -d CNFS -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f Eurostat -d nama_10_a10 -S
dlstats fetchers tag -f OECD -d MEI -S
"""
ctx = client.Context(**kwargs)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
start = time.time()
db = ctx.mongo_database()
if fetcher:
fetchers = [fetcher]
else:
fetchers = FETCHERS.keys()
for provider in fetchers:
provider_name = provider
ctx.log("START update tags for [%s]" % provider_name)
"""
ctx.log("Update provider[%s] Categories tags..." % provider_name)
try:
result = tags.update_tags_categories(db,
provider_name=provider_name,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_ok("Update provider[%s] Categories tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Categories tags Fail - provider[%s] - [%s]" % (provider_name, str(err)))
"""
ctx.log("Update provider[%s] Datasets tags..." % provider_name)
try:
result = tags.update_tags_datasets(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_ok("Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err)))
ctx.log("Update provider[%s] Series tags..." % provider_name)
try:
result = tags.update_tags_series(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
async_mode=async_mode,
dry_mode=dry_mode)
if not async_mode:
ctx.log_ok("Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception:
ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error()))
end = time.time() - start
ctx.log("END update tags for [%s] - time[%.3f]" % (provider_name, end))
def test_update_tags_series(self):
# nosetests -s -v widukind_common.tests.test_tags:UpdateTagsTestCase.test_update_tags_series
self.db[constants.COL_PROVIDERS].insert(self.doc_provider)
self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1)
dataset = {
"enable": True,
"provider_name": self.doc_provider["name"],
"dataset_code": "d1",
"name": "dataset 1",
"slug": "%s-d1" % self.doc_provider["slug"],
"concepts": {
"FREQ": "Frequency",
"OBS_STATUS": "Observation Status",
"COUNTRY": "Country"
},
"codelists": {
"FREQ": {
"D": "Daily",
"M": "Monthly"
},
"OBS_STATUS": {
"E": "Estimate",
},
"COUNTRY": {
"FRA": "France"
}
},
"dimension_keys": ["FREQ", "COUNTRY"],
"attribute_keys": ["OBS_STATUS"],
}
self.db[constants.COL_DATASETS].insert(dataset)
self.assertEqual(self.db[constants.COL_DATASETS].count(), 1)
series = {
"provider_name": self.doc_provider["name"],
"dataset_code": dataset["dataset_code"],
"key": "x1",
"name": "series 1",
"slug": "%s-d1-x1" % self.doc_provider["slug"],
"frequency": "M",
"dimensions": {
"COUNTRY": "FRA"
},
"attributes": {
"OBS_STATUS": "E"
},
}
self.db[constants.COL_SERIES].insert(series)
self.assertEqual(self.db[constants.COL_SERIES].count(), 1)
result = tags_utils.update_tags_series(self.db,
self.doc_provider["name"],
dataset["dataset_code"])
self.assertEqual(result["nMatched"], 1)
self.assertEqual(result["nModified"], 1)
result = tags_utils.update_tags_series(self.db,
self.doc_provider["name"],
dataset["dataset_code"],
update_only=True)
self.assertEqual(result["nMatched"], 0)
self.assertEqual(result["nModified"], 0)
query = {
"provider_name": series["provider_name"],
"dataset_code": series["dataset_code"],
"key": series["key"]
}
series_doc = self.db[constants.COL_SERIES].find_one(query)
self.assertIsNotNone(series_doc)
self.assertEqual(series_doc["tags"], [
'country', 'd1', 'dataset', 'estimate', 'france', 'mars',
'monthly', 'observation', 'p1', 'provider', 'series', 'status',
'test', 'x1'
])
