def test_search_datasets_tags(self): # nosetests -s -v widukind_common.tests.test_tags:SearchTagsTestCase.test_search_datasets_tags self.db[constants.COL_PROVIDERS].insert(self.doc_provider) self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1) self.db[constants.COL_DATASETS].insert(self.doc_dataset) self.assertEqual(self.db[constants.COL_DATASETS].count(), 1) tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"]) query = { "provider_name": self.doc_dataset["provider_name"], "dataset_code": self.doc_dataset["dataset_code"] } dataset_doc = self.db[constants.COL_DATASETS].find_one(query) self.assertEqual(dataset_doc["tags"], [ 'country', 'd1', 'daily', 'dataset', 'estimate', 'france', 'frequency', 'mars', 'monthly', 'observation', 'p1', 'provider', 'status', 'test' ]) cursor, query = tags_utils.search_datasets_tags( self.db, search_tags="France MARS Daily") self.assertEqual(cursor.count(), 1) cursor, query = tags_utils.search_datasets_tags( self.db, search_tags="France MARS Daily", provider_name="UNKNOW") self.assertEqual(cursor.count(), 0) """
def test_search_datasets_tags(self): # nosetests -s -v widukind_common.tests.test_tags:SearchTagsTestCase.test_search_datasets_tags self.db[constants.COL_PROVIDERS].insert(self.doc_provider) self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1) self.db[constants.COL_DATASETS].insert(self.doc_dataset) self.assertEqual(self.db[constants.COL_DATASETS].count(), 1) tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"]) query = {"provider_name": self.doc_dataset["provider_name"], "dataset_code": self.doc_dataset["dataset_code"]} dataset_doc = self.db[constants.COL_DATASETS].find_one(query) self.assertEqual(dataset_doc["tags"], ['country', 'd1', 'daily', 'dataset', 'estimate', 'france', 'frequency', 'mars', 'monthly', 'observation', 'p1', 'provider', 'status', 'test']) cursor, query = tags_utils.search_datasets_tags(self.db, search_tags="France MARS Daily") self.assertEqual(cursor.count(), 1) cursor, query = tags_utils.search_datasets_tags(self.db, search_tags="France MARS Daily", provider_name="UNKNOW") self.assertEqual(cursor.count(), 0) """
def test_update_tags_datasets(self): # nosetests -s -v widukind_common.tests.test_tags:UpdateTagsTestCase.test_update_tags_datasets self.db[constants.COL_PROVIDERS].insert(self.doc_provider) self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1) dataset = { "enable": True, "provider_name": self.doc_provider["name"], "dataset_code": "d1", "name": "dataset 1", "slug": "%s-d1" % self.doc_provider["slug"], "concepts": { "FREQ": "Frequency", "OBS_STATUS": "Observation Status", }, "codelists": { "FREQ": { "D": "Daily" }, "OBS_STATUS": { "E": "Estimate" } }, "dimension_keys": ["FREQ"], "attribute_keys": ["OBS_STATUS"], } self.db[constants.COL_DATASETS].insert(dataset) self.assertEqual(self.db[constants.COL_DATASETS].count(), 1) result = tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], dataset["dataset_code"]) self.assertEqual(len(result["writeErrors"]), 0) self.assertEqual(result["nMatched"], 1) self.assertEqual(result["nModified"], 1) result = tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], dataset["dataset_code"], update_only=True) self.assertEqual(result["nMatched"], 0) self.assertEqual(result["nModified"], 0) query = { "provider_name": dataset["provider_name"], "dataset_code": dataset["dataset_code"] } dataset_doc = self.db[constants.COL_DATASETS].find_one(query) self.assertIsNotNone(dataset_doc) self.assertEqual(dataset_doc["tags"], [ 'd1', 'daily', 'dataset', 'estimate', 'frequency', 'mars', 'observation', 'p1', 'provider', 'status', 'test' ])
def test_update_tags_datasets(self): # nosetests -s -v widukind_common.tests.test_tags:UpdateTagsTestCase.test_update_tags_datasets self.db[constants.COL_PROVIDERS].insert(self.doc_provider) self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1) dataset = { "enable": True, "provider_name": self.doc_provider["name"], "dataset_code": "d1", "name": "dataset 1", "slug": "%s-d1" % self.doc_provider["slug"], "concepts": { "FREQ": "Frequency", "OBS_STATUS": "Observation Status", }, "codelists": { "FREQ": { "D": "Daily" }, "OBS_STATUS": { "E": "Estimate" } }, "dimension_keys": ["FREQ"], "attribute_keys": ["OBS_STATUS"], } self.db[constants.COL_DATASETS].insert(dataset) self.assertEqual(self.db[constants.COL_DATASETS].count(), 1) result = tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], dataset["dataset_code"]) self.assertEqual(len(result["writeErrors"]), 0) self.assertEqual(result["nMatched"], 1) self.assertEqual(result["nModified"], 1) result = tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], dataset["dataset_code"], update_only=True) self.assertEqual(result["nMatched"], 0) self.assertEqual(result["nModified"], 0) query = {"provider_name": dataset["provider_name"], "dataset_code": dataset["dataset_code"]} dataset_doc = self.db[constants.COL_DATASETS].find_one(query) self.assertIsNotNone(dataset_doc) self.assertEqual(dataset_doc["tags"], ['d1', 'daily', 'dataset', 'estimate', 'frequency', 'mars', 'observation', 'p1', 'provider', 'status', 'test'])
def test_search_series_tags(self): # nosetests -s -v widukind_common.tests.test_tags:SearchTagsTestCase.test_search_series_tags self.db[constants.COL_PROVIDERS].insert(self.doc_provider) self.assertEqual(self.db[constants.COL_PROVIDERS].count(), 1) self.db[constants.COL_DATASETS].insert(self.doc_dataset) self.assertEqual(self.db[constants.COL_DATASETS].count(), 1) tags_utils.update_tags_datasets(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"]) self.db[constants.COL_SERIES].insert(self.doc_series) self.assertEqual(self.db[constants.COL_SERIES].count(), 1) tags_utils.update_tags_series(self.db, self.doc_provider["name"], self.doc_dataset["dataset_code"])
def _update_tags(ctx, db, provider_name, dataset=None, max_bulk=100, update_only=False, dry_mode=False, async_mode=None): start = time.time() ctx.log("START update tags for [%s]" % provider_name) ctx.log("Update provider[%s] Datasets tags..." % provider_name) try: result = tags.update_tags_datasets(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, dry_mode=dry_mode) ctx.log_warn("Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception as err: ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err))) ctx.log("Update provider[%s] Series tags..." % provider_name) try: result = tags.update_tags_series(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, async_mode=async_mode, dry_mode=dry_mode) if not async_mode: ctx.log_warn("Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception: ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error())) end = time.time() - start ctx.log("update tags END: provider[%s] - time[%.3f]" % (provider_name, end))
def _update_tags(ctx, db, provider_name, dataset=None, max_bulk=100, update_only=False, dry_mode=False, async_mode=None): start = time.time() ctx.log("START update tags for [%s]" % provider_name) ctx.log("Update provider[%s] Datasets tags..." % provider_name) try: result = tags.update_tags_datasets(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, dry_mode=dry_mode) ctx.log_warn( "Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception as err: ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err))) ctx.log("Update provider[%s] Series tags..." % provider_name) try: result = tags.update_tags_series(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, async_mode=async_mode, dry_mode=dry_mode) if not async_mode: ctx.log_warn( "Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception: ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error())) end = time.time() - start ctx.log("update tags END: provider[%s] - time[%.3f]" % (provider_name, end))
def cmd_update_tags(fetcher=None, dataset=None, max_bulk=100, update_only=False, async_mode=None, dry_mode=False, **kwargs): """Create or Update field tags""" """ Examples: dlstats fetchers tag -f BIS -d CNFS -S dlstats fetchers tag -f BEA -d "10101 Ann" -S dlstats fetchers tag -f BEA -d "10101 Ann" -S dlstats fetchers tag -f Eurostat -d nama_10_a10 -S dlstats fetchers tag -f OECD -d MEI -S """ ctx = client.Context(**kwargs) if ctx.silent or click.confirm('Do you want to continue?', abort=True): start = time.time() db = ctx.mongo_database() if fetcher: fetchers = [fetcher] else: fetchers = FETCHERS.keys() for provider in fetchers: provider_name = provider ctx.log("START update tags for [%s]" % provider_name) """ ctx.log("Update provider[%s] Categories tags..." % provider_name) try: result = tags.update_tags_categories(db, provider_name=provider_name, max_bulk=max_bulk, update_only=update_only, dry_mode=dry_mode) ctx.log_ok("Update provider[%s] Categories tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception as err: ctx.log_error("Update Categories tags Fail - provider[%s] - [%s]" % (provider_name, str(err))) """ ctx.log("Update provider[%s] Datasets tags..." % provider_name) try: result = tags.update_tags_datasets(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, dry_mode=dry_mode) ctx.log_ok("Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception as err: ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err))) ctx.log("Update provider[%s] Series tags..." % provider_name) try: result = tags.update_tags_series(db, provider_name=provider_name, dataset_code=dataset, max_bulk=max_bulk, update_only=update_only, async_mode=async_mode, dry_mode=dry_mode) if not async_mode: ctx.log_ok("Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"])) except Exception: ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error())) end = time.time() - start ctx.log("END update tags for [%s] - time[%.3f]" % (provider_name, end))