def upsert_data_tree(self, data_tree=None, force_update=False):
if data_tree and not isinstance(data_tree, list):
raise TypeError("data_tree is not instance of list")
if not data_tree:
data_tree = self.build_data_tree(force_update=force_update)
self.provider.data_tree = data_tree
for d in data_tree:
schemas.data_tree_schema(d)
query_filter = {'name': self.provider_name}
query_update = {'$set': {'data_tree': data_tree}}
return self.db[constants.COL_PROVIDERS].find_one_and_update(query_filter,
query_update,
return_document=pymongo.ReturnDocument.AFTER)
def test_build_data_tree(self):
# nosetests -s -v dlstats.tests.fetchers.test_ecb:FetcherTestCase.test_build_data_tree
self._register_urls_data_tree()
self.fetcher.build_data_tree()
# self.maxDiff = None
provider = self.fetcher.provider
self.assertEqual(provider.count_data_tree(), 12)
"""
pprint(provider.data_tree)
with open(DATA_TREE_FP, "w") as fp:
json.dump(provider.data_tree, fp, sort_keys=False)
"""
new_provider = Providers(fetcher=self.fetcher, **provider.bson)
with open(DATA_TREE_FP) as fp:
local_data_tree = json.load(fp, object_pairs_hook=OrderedDict)
new_provider.data_tree = local_data_tree
# self.assertEqual(provider.data_tree, new_provider.data_tree)
filter_datasets = provider.datasets(category_filter="ECB.MOBILE_NAVI.06")
self.assertEqual(len(filter_datasets), 6)
self.assertEqual(filter_datasets[0]["dataset_code"], "BOP")
self.assertEqual(filter_datasets[-1]["dataset_code"], "TRD")
for d in provider.data_tree:
schemas.data_tree_schema(d)
provider.update_database()
doc = self.db[constants.COL_PROVIDERS].find_one({"name": self.fetcher.provider_name})
self.assertIsNotNone(doc)
for i, d in enumerate(doc["data_tree"]):
self.assertEqual(doc["data_tree"][i], provider.data_tree[i])
count = len(self.fetcher.datasets_list())
self.assertEqual(count, DATAFLOW_COUNT)