Exemple #1
0
    def upsert_data_tree(self, data_tree=None, force_update=False):

        if data_tree and not isinstance(data_tree, list):
            raise TypeError("data_tree is not instance of list")

        if not data_tree:
            data_tree = self.build_data_tree(force_update=force_update)

        self.provider.data_tree = data_tree

        for d in data_tree:
            schemas.data_tree_schema(d)

        query_filter = {'name': self.provider_name}
        query_update = {'$set': {'data_tree': data_tree}}

        return self.db[constants.COL_PROVIDERS].find_one_and_update(query_filter,
                                                                    query_update,
                                                                    return_document=pymongo.ReturnDocument.AFTER)
Exemple #2
0
    def test_build_data_tree(self):

        # nosetests -s -v dlstats.tests.fetchers.test_ecb:FetcherTestCase.test_build_data_tree

        self._register_urls_data_tree()

        self.fetcher.build_data_tree()

        # self.maxDiff = None

        provider = self.fetcher.provider
        self.assertEqual(provider.count_data_tree(), 12)

        """
        pprint(provider.data_tree)
        with open(DATA_TREE_FP, "w") as fp:
            json.dump(provider.data_tree, fp, sort_keys=False)
        """

        new_provider = Providers(fetcher=self.fetcher, **provider.bson)

        with open(DATA_TREE_FP) as fp:
            local_data_tree = json.load(fp, object_pairs_hook=OrderedDict)
            new_provider.data_tree = local_data_tree
            # self.assertEqual(provider.data_tree, new_provider.data_tree)

        filter_datasets = provider.datasets(category_filter="ECB.MOBILE_NAVI.06")
        self.assertEqual(len(filter_datasets), 6)
        self.assertEqual(filter_datasets[0]["dataset_code"], "BOP")
        self.assertEqual(filter_datasets[-1]["dataset_code"], "TRD")

        for d in provider.data_tree:
            schemas.data_tree_schema(d)

        provider.update_database()

        doc = self.db[constants.COL_PROVIDERS].find_one({"name": self.fetcher.provider_name})
        self.assertIsNotNone(doc)
        for i, d in enumerate(doc["data_tree"]):
            self.assertEqual(doc["data_tree"][i], provider.data_tree[i])

        count = len(self.fetcher.datasets_list())
        self.assertEqual(count, DATAFLOW_COUNT)