def test_ingest(self, mock_get):
identifier = 'hvd.32044106431737'
side_effects = []
with open('cookies/tests/data/hathitrust_brief_volume_metadata.json'
) as f:
side_effects.append(MockDataResponse(200, json.load(f)))
with open('cookies/tests/data/hathitrust_volume_metadata.json') as f:
side_effects.append(MockDataResponse(200, json.load(f)))
mock_get.side_effect = side_effects
ingest = IngesterFactory().get(
'cookies.accession.hathitrust.HathiTrustRemoteIngest')(
[identifier], 'asdf', '1234', 'http://asdf.com',
'http://asdf.com')
data = ingest.next()
self.assertIsInstance(data, Resource)
self.assertEqual(ResourceContainer.objects.count(), 1)
self.assertEqual(
Resource.objects.filter(content_resource=False).count(), 61)
self.assertEqual(
Resource.objects.filter(content_resource=True).count(), 60)
for resource in Resource.objects.filter(content_resource=True):
self.assertEqual(resource.content_type, 'text/plain')
for relation in ContentRelation.objects.all():
self.assertEqual(relation.content_type, 'text/plain')
def test_ingest(self):
factory = IngesterFactory()
ingest_class = factory.get('cookies.accession.zotero.ZoteroIngest')
ingester = ingest_class("test_data/TestRDF.rdf")
ingester.set_resource_defaults(**self.resource_data)
N = 0
for resource in ingester:
self.assertIsInstance(resource, Resource)
N += 1
self.assertEqual(N, 20, "Should create 20 resources from this RDF.")
def test_ingest(self):
factory = IngesterFactory()
ingest_class = factory.get('cookies.accession.zotero.ZoteroIngest')
ingester = ingest_class("test_data/TestRDF.zip")
ingester.set_resource_defaults(**self.resource_data)
N = 0
for resource in ingester:
self.assertIsInstance(resource, Resource)
self.assertGreater(
resource.content.count(), 0,
"Each resource in this RDF should have some form of content.")
N += 1
self.assertEqual(N, 20, "Should create 20 resources from this RDF.")
class TestImport(unittest.TestCase):
def setUp(self):
self.factory = IngesterFactory()
disconnect_signal(post_save, send_all_files_to_giles, ContentRelation)
def test_import_factory(self):
"""
The IngesterFactory should return a wrapped object that supports
iteration. Each iteration should yield a :class:`.Resource` instance.
"""
ingest_class = self.factory.get('cookies.tests.mocks.MockIngester')
mock_file = MockFile()
mock_file.read = mock.MagicMock(name='read')
mock_file.read.return_value = [
{
'name': 'Test',
},
{
'name': 'Test2',
},
]
ingester = ingest_class(mock_file)
self.assertIsInstance(ingester, IngestManager)
self.assertIsInstance(ingester.next(), Resource)
self.assertEqual(mock_file.read.call_count, 1)
def tearDown(self):
reconnect_signal(post_save, send_all_files_to_giles, ContentRelation)
def setUp(self):
self.factory = IngesterFactory()
def handle_bulk(self,
file_path,
form_data,
file_name,
job=None,
ingester='cookies.accession.zotero.ZoteroIngest'):
"""
Process resource data in a RDF document.
Parameters
----------
file_path : str
Local path to a RDF document, or a ZIP file containing a Zotero RDF
export (with files).
form_data : dict
Valid data from a :class:`cookies.forms.BulkResourceForm`\.
file_name : str
Name of the target file at ``file_path``\.
job : :class:`.UserJob`
Used to update progress.
"""
if job:
job.result_id = self.request.id
job.save()
logger.debug('handle bulk')
creator = form_data.pop('created_by')
# The user can either add these new records to an existing collection, or
# create a new one.
collection = form_data.pop('collection', None)
collection_name = form_data.pop('name', None)
if not collection:
collection = Collection.objects.create(**{
'name': collection_name,
'created_by': creator,
})
operations.add_creation_metadata(collection, creator)
# User can indicate a default Type to assign to each new Resource.
default_type = form_data.pop('default_type', None)
ingester = IngesterFactory().get(ingester)(file_path)
ingester.Resource = authorization.apply_filter(creator, 'change_resource',
ingester.Resource)
ingester.Collection = authorization.apply_filter(creator,
'change_collection',
ingester.Collection)
ingester.ConceptEntity = authorization.apply_filter(
creator, 'change_conceptentity', ingester.ConceptEntity)
ingester.set_resource_defaults(entity_type=default_type,
created_by=creator,
**form_data)
N = len(ingester)
for resource in ingester:
collection.resources.add(resource)
operations.add_creation_metadata(resource, creator)
authorization.update_authorizations(Resource.DEFAULT_AUTHS,
creator,
resource,
propagate=True)
if job:
job.progress += 1. / N
job.save()
job.result = jsonpickle.encode({'view': 'collection', 'id': collection.id})
job.save()
return {'view': 'collection', 'id': collection.id}
def handle_bulk(self,
file_path,
form_data,
file_name,
job=None,
ingester='cookies.accession.zotero.ZoteroIngest'):
"""
Process resource data in a RDF document.
Parameters
----------
file_path : str
Local path to a RDF document, or a ZIP file containing a Zotero RDF
export (with files).
form_data : dict
Valid data from a :class:`cookies.forms.BulkResourceForm`\.
file_name : str
Name of the target file at ``file_path``\.
job : :class:`.UserJob`
Used to update progress.
"""
if job:
job.result_id = self.request.id
job.save()
logger.debug('handle bulk')
creator = form_data.pop('created_by')
# The user can either add these new records to an existing collection, or
# create a new one.
collection = form_data.pop('collection', None)
collection_name = form_data.pop('name', None)
public_policy = False
if not collection:
collection = Collection.objects.create(**{
'name': collection_name,
'created_by': creator,
})
operations.add_creation_metadata(collection, creator)
if form_data.get('public'):
# Create an authoirzation for AnonymousUser.
CollectionAuthorization.objects.create(
granted_by=creator,
granted_to=None,
for_resource=collection,
policy=CollectionAuthorization.ALLOW,
action=CollectionAuthorization.VIEW)
public_policy = True
# User can indicate a default Type to assign to each new Resource.
default_type = form_data.pop('default_type', None)
upload_resource = Resource.objects.create(
created_by=creator,
name=file_name,
)
with open(file_path, 'r') as f:
upload_resource.file.save(file_name, File(f), True)
ingester = IngesterFactory().get(ingester)(upload_resource.file.path)
ingester.Resource = authorization.apply_filter(ResourceAuthorization.EDIT,
creator, ingester.Resource)
ingester.Collection = authorization.apply_filter(
ResourceAuthorization.EDIT, creator, ingester.Collection)
ingester.ConceptEntity = authorization.apply_filter(
ResourceAuthorization.EDIT, creator, ingester.ConceptEntity)
ingester.set_resource_defaults(entity_type=default_type,
collection=collection,
created_by=creator,
**form_data)
ingester.set_collection(collection)
N = len(ingester)
resource_auths = []
for resource in ingester:
resource.container.part_of = collection
if form_data.get('public') and not public_policy:
resource_auths.append(
ResourceAuthorization(granted_by=creator,
granted_to=None,
for_resource=resource.container,
policy=ResourceAuthorization.ALLOW,
action=ResourceAuthorization.VIEW))
resource.container.save()
# collection.resources.add(resource)
operations.add_creation_metadata(resource, creator)
if job:
job.progress += 1. / N
job.save()
ResourceAuthorization.objects.bulk_create(resource_auths)
job.result = jsonpickle.encode({'view': 'collection', 'id': collection.id})
job.save()
return {'view': 'collection', 'id': collection.id}
def setUp(self):
self.factory = IngesterFactory()
disconnect_signal(post_save, send_all_files_to_giles, ContentRelation)