def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras["harvest:domain"] = "test.org"
dataset.extras["harvest:remote_id"] = str(i)
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({"local": str(dataset.id), "remote": str(index)})
csvfile.flush()
result = actions.attach("test.org", csvfile.name)
dbcount = Dataset.objects(**{"extras__harvest:remote_id__exists": True}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras["harvest:domain"], "test.org")
self.assertEqual(dataset.extras["harvest:remote_id"], str(index))
def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras['harvest:domain'] = 'test.org'
dataset.extras['harvest:remote_id'] = str(i)
dataset.last_modified = datetime.now()
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
dbcount = Dataset.objects(**{
'extras__harvest:remote_id__exists': True
}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras['harvest:domain'] = 'test.org'
dataset.extras['harvest:remote_id'] = str(i)
dataset.last_modified = datetime.now()
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
dbcount = Dataset.objects(**{
'extras__harvest:remote_id__exists': True
}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def process(self, item):
mock_process.send(self, item=item)
return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
def process(self, item):
mock_process.send(self, item=item)
return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))