def documents(self):
"""
Iterates over the collection using the filter condition and yields each document in turn.
"""
with get_mongo(self.config, collection=self.mongo_collection) as mongo:
for document in mongo.find(self.condition):
yield document
def records(self):
'''
Generator function which yields DeletionRecord objects.
:return: yields DeletionRecords
'''
with get_mongo(common.CONFIG, collection=self.resource_id) as mongo:
# loop through records in mongo
for record in mongo.find(projection=[u'id', u'data']):
# only delete the record if it hasn't already been deleted
if record[u'data']:
yield DeletionRecord(self.version, self.resource_id,
record[u'id'])
def get_existing_max_id(self):
'''
Figure out what the current max id is in this resource's collection.
:return: the highest id in the collection currently (it'll be an int), or None if there
aren't any documents in the collection
'''
with get_mongo(self.config, collection=self.resource_id) as mongo:
# sort by id descending to get the highest
doc_with_max_id = mongo.find_one(sort=[(u'id', -1)])
# find_one returns None if there aren't any matching documents
if doc_with_max_id is None:
return None
else:
return doc_with_max_id[u'id']
def records(self):
'''
Generator function which yields DeletionRecord objects. When the replace flag is true during
ingestion this indicates that any records not present in the new resource data should be
deleted. By using the tracker this feeder can yield record objects which represent a record
that was not included in a new version of a resource's data for deletion.
:return: yields DeletionRecords
'''
with get_mongo(common.CONFIG, collection=self.resource_id) as mongo:
# this finds all the records that haven't been updated in the given version
for mongo_doc in mongo.find(
{u'latest_version': {
u'$lt': self.version
}}):
if not self.tracker.was_included(mongo_doc[u'id']):
# delete
yield DeletionRecord(self.version, self.resource_id,
mongo_doc[u'id'])
def test_get_with_config_and_database_and_collection(self):
with get_mongo(TestMongo.config,
database=u'test_database',
collection=u'test_collection') as mongo:
assert type(mongo) is Collection
def test_get_with_config_and_database(self):
with get_mongo(TestMongo.config, database=u'test_database') as mongo:
assert type(mongo) is Database
def test_get_with_just_config(self):
with get_mongo(TestMongo.config) as mongo:
assert type(mongo) is MongoClient
def total(self):
"""
Counts and returns the number of documents which will match the condition.
"""
with get_mongo(self.config, collection=self.mongo_collection) as mongo:
return mongo.count_documents(self.condition)