def test_default_value_date_on_add(self):
scan_data = {'archive_id': 3, 'archiveFile': 'a_repo'}
then = now()
result = self.add_one_scan(scan_data).json
self.assertTrue(result['date'])
self.assertMoreRecent(result['date'], then)
# these two dates should be approximately the same (give or take a millisecond)
self.assertEqual(result['date'][:20], result['dateLastModified'][:20])
a_date = string_to_datetime('2012-01-01')
scan_data = {'archive_id': 3, 'archiveFile': 'a_repo', 'date': a_date}
then = now()
result = self.add_one_scan(scan_data).json
self.assertEqual(string_to_datetime(result['date']), a_date)
def test_update_translationID(self):
"translationID is correctly updated"
result = self.add_one_scan(self.scan_data).json
self.assertEqual(result.get('translationID'), None)
url = localurl(result['URL'])
then = now()
self.app.put(url, {'translationID': 'Some new value'})
result = self.app.get(url).json
self.assertEqual(result['translationID'], 'Some new value')
self.assertMoreRecent(result['dateLastModified'], then)
def test_update_status(self):
"changing status works only for valid values"
result = self.add_one_scan(self.scan_data).json
self.assertEqual(result['status'], 1)
url = localurl(result['URL'])
then = now()
self.app.put(url, {'status': 2})
result = self.app.get(url).json
self.assertEqual(result['status'], 2)
self.assertMoreRecent(result['dateLastModified'], then)
def test_get_archive_info(self):
filecontents = self.get_default_filecontents()
then = now()
res = self.add_one_ead(filecontents=filecontents, dontlog=True).json
self.assertEqual(res['institution'], self.default_institution)
self.assertEqual(res['archive'], self.default_archive)
self.assertMoreRecent(res['dateLastModified'], then)
self.assertEqual(res['language'], self.default_language)
self.assertEqual(res['findingaid'], 'FindingAid')
# if we change the filecontents, we should see that reflected in the reult
filecontents = filecontents.replace(self.default_institution,
'ID-JaAN')
filecontents = filecontents.replace(self.default_archive, 'Krawang')
res = self.change_ead(filecontents=filecontents).json
self.assertEqual(res['archive'], 'Krawang')
def test_ead_update(self):
filecontents = self.get_default_filecontents()
res = self.add_one_ead(filecontents=filecontents)
url = localurl(res.json['URL'])
TO_REPLACE = 'UNITTITLE'
self.assertTrue(TO_REPLACE in filecontents)
newfilecontents = filecontents.replace(TO_REPLACE, 'changed_string')
filetuple = ('file', 'test_file_123.xml', newfilecontents)
then = now()
res = self.app.put(url,
upload_files=[filetuple],
extra_environ={'dontlog_web_chats': '1'})
self.assertMoreRecent(res.json['dateLastModified'], then)
ead = self.app.get(url + '/file',
extra_environ={'dontlog_web_chats': '1'})
self.assertEqual(ead.content_type, 'text/xml')
self.assertEqual(ead.body, newfilecontents)
def put_scan_image_item(request):
"""
Change this image
* **file: optional**
* **is_default:**
optional
if this is True, then the current image will become the default image.
"""
is_default = request.validated['is_default']
scan = request._dbentity['scan']
scanimage = request._dbentity['scanimage']
if is_default:
for image in scan.images:
image.is_default = False
scanimage.is_default = True
if 'file' in request.POST:
scan.delete_files_for_image(scanimage.id)
scanimage.filename = request.POST['file'].filename
scan.store_file(request.POST['file'].file.read(), scanimage.id)
# updae the last_modified date of the scan
scan.last_modified = now()
# TOOD: next line can be optimized (using partial_update_keys)
request.solr_scan.update([scan.get_solr_data()])
user = get_user(request)
log_events(request.db, user, [{
'message': 'update',
'object_id': scan.number,
'object_type': 'scan'
}])
log_events(request.db, user, [{
'message': 'update',
'object_id': scanimage.id,
'object_type': 'image'
}])
return {'success': True, 'image': dict(scanimage)}
def delete_scan_image(request):
"""
Delete this image.
if successfull, returns {"success": "True"}
"""
image = request._dbentity['scanimage']
if image.is_default:
for candidate in image.scan.images:
if candidate.id != image.id:
candidate.is_default = True
scan = image.scan
scan.last_modified = now()
db = request.db
image_id = image.id
scan.delete_files_for_image(image_id)
db.delete(image)
db.flush()
db.refresh(scan)
# TOOD: next line can be optimized (using partial_update_keys)
scan_solr_data = scan.get_solr_data()
request.solr_scan.update([scan_solr_data])
# TOOD: next line can be optimized (using partial_update_keys)
archivefile = cast_scan_as_archivefile(
request, scan_solr_data).get_solr_data(request)
archivefile['id'] = archivefile['archivefile_id']
request.solr_archivefile.update([archivefile])
# delete the archive file if it remains orphaned
delete_orphaned_archivefile(request,
archivefile,
check_for_db_record=False)
user = get_user(request)
log_events(request.db, user, [{
'message': 'delete',
'object_id': image_id,
'object_type': 'image'
}])
return {"success": "True"}
def add_ead_file(
context,
name,
filecontents,
status=1,
):
"""Add an EAD file
NB: No validation, No logging, use with care!
(validation and loggging is done in browser.ead.service_add_ead_file)
"""
eadfile = EadFile(context=context)
eadfile.name = name
eadfile.status = status
store_file(
eadfile.get_file_path(),
filecontents,
)
context.db.add(eadfile)
eadfile.last_modified = now()
return eadfile
def add_scan_image(request):
"""Add one or more images to the collection
* **file:**
the file binary data; if repeated many images will be created
* **is_default:**
if set to non-empty, non-zero string
this will become the default image.
If many images are passed in the file parameters,
the first will be the default one.
"""
is_default = request.validated['is_default']
scan = request._dbentity['scan']
scan.last_modified = now()
if is_default:
for image in scan.images:
image.is_default = False
added_images = add_files_from_request(request, scan, is_default=is_default)
# TOOD: next line can be optimized (using partial_update_keys)
request.solr_scan.update([scan.get_solr_data()])
user = get_user(request)
for image in added_images:
log_events(request.db, user, [{
'message': 'create',
'object_id': image.id,
'object_type': 'image'
}])
return {
'success': True,
'results': [dict(image) for image in added_images]
}
def update_ead_file(request):
"""
Update an EAD file
parameters:
* **file:**
the XML file to
* **user:**
the name of a user - optional, will be used for logging info
%(PARAM_STATUS)s
returns: JSON object representing the EAD file
see :ref:`TestEad.test_ead_update`
"""
eadfile=request._dbentity['ead']
ead_id=eadfile.name
path=eadfile.get_file_path()
if 'raw_file' in request.validated:
store_file(path, request.validated['raw_file'])
if 'status' in request.POST:
eadfile.status=request.validated['status']
user=get_user(request)
if 'raw_file' in request.validated or eadfile in request.db.dirty:
eadfile.last_modified=now()
log_events(request.db, user, [{
'object_id': ead_id,
'object_type': 'ead',
'message': 'update'
}])
request.solr_ead.update([eadfile.get_solr_data()])
request.solr.commit() # we need to commit before we index the archivefile
request.solr_eadcomponent.delete_by_query('ead_id:' + ead_id)
components = eadfile.extract_component_dicts()
request.solr_eadcomponent.update(components)
request.solr.commit()
# archivefiles that are already indexed
indexed_archivefiles = request.solr_archivefile.search(q='ead_ids:' + eadfile.name).documents
# components of this ead file that are to be indexed as archivefiles
to_index_archivefiles = [cast_component_as_archivefile(x).get_solr_data(request) for x in components if x['is_archiveFile']]
to_index_archivefile_ids = [x['archivefile_id'] for x in to_index_archivefiles]
to_delete_archivefiles = []
for x in indexed_archivefiles:
# delete all archivefiles that have no scans available, and are not in this ead file (anymore)
if x['number_of_scans'] == 0 and x['archivefile_id'] not in to_index_archivefile_ids:
request.solr_archivefile.delete_by_key(x['id'])
to_delete_archivefiles.append(x)
db_records = get_archivefiles(request, archive_id=x['archive_id'])
db_records = {db_record.id: db_record for db_record in db_records}
for document in to_index_archivefiles:
if document['archivefile_id'] in db_records:
db_record = db_records[document['archivefile_id']]
if document['status'] != db_record.status:
document['status'] = db_record.status
request.solr_archivefile.update(to_index_archivefiles)
# commit, and ping the pagebrowser
for x in to_delete_archivefiles:
pagebrowser.update.delete_book(request, ead_id=ead_id, archivefile_id=x['archivefile_id'])
for document in to_index_archivefiles:
if document['archivefile_id'] in db_records:
if document['status'] == STATUS_PUBLISHED:
pagebrowser.update.refresh_book(request, ead_id=ead_id, archivefile_id=document['archivefile_id'])
else:
pagebrowser.update.delete_book(request, ead_id=ead_id, archivefile_id=document['archivefile_id'])
return eadfile.to_dict(request)
def test_add_lastmodified(self):
scan_data = {'archive_id': 3, 'archiveFile': 'a_repo'}
then = now()
result = self.add_one_scan(scan_data).json
self.assertEqual(result['archive_id'], 3)
self.assertMoreRecent(result['dateLastModified'], then)
def update_scan(context, scan, data, filecontents, filename, user=None):
"""
- update solr data of scan
- update solr data of archivefile
- update solr data of eadcomponent
- add log entry
"""
db = context.db
old_archivefile = dict([(field, getattr(scan, field))
for field in ARCHIVE_IDS])
sa_old_condition = get_archivefile_condition(scan)
for key in data:
setattr(scan, key, data[key])
scan.last_modified = now()
# we do a partial update for optimization
scan_solr_data = scan.get_solr_data(partial_update_keys=data.keys())
context.solr_scan.update([scan_solr_data])
# context.solr.commit(soft_commit=OPTIMIZATION_SOFT_COMMIT)
new_archivefile = dict([(field, getattr(scan, field))
for field in ARCHIVE_IDS])
if old_archivefile != new_archivefile:
# we changed the archiveFile to which the scan belongs
# and so we need to move a lot of stuff around
sa_new_condition = get_archivefile_condition(scan)
old_sequenceNumber = scan.sequenceNumber
# We must update sequenceNumbers. First null current scan and flush
# scan.sequenceNumber = None
# db.flush()
query = sqlalchemy.func.max(Scan.sequenceNumber)
highest_number = db.query(query).filter(sa_new_condition).one()[0] or 0
scan.sequenceNumber = highest_number + 1
# Now patch the hole in the old scan collection
logging.debug('updating sequence numbers')
to_decrease = sa_old_condition & (Scan.sequenceNumber >
old_sequenceNumber)
elements = db.query(Scan).filter(to_decrease)
materialized_elements = elements.all()
elements.update({Scan.sequenceNumber: Scan.sequenceNumber - 1})
logging.debug('done')
# now the database is updated, we update solr
logging.debug('updating solr data')
# materialized_elements = elements
materialized_elements += [scan]
solr_objects = [{
'number': el.number,
'sequenceNumber': {
'set': el.sequenceNumber
}
} for el in materialized_elements]
context.solr_scan.update(solr_objects)
logging.debug('done')
# since we changed the archive file, we also need to update the scan counts in the
# corresponding components
old_components = search_components(context=context,
with_facets=False,
limit=1,
**old_archivefile)['results']
if old_components:
old_component = old_components[0]
context.solr_eadcomponent.update([{
'eadcomponent_id':
old_component['eadcomponent_id'],
'number_of_scans': {
'set': old_component['number_of_scans'] - 1
}
}])
new_components = search_components(context=context,
with_facets=False,
limit=1,
**new_archivefile)['results']
if new_components:
new_component = new_components[0]
context.solr_eadcomponent.update([{
'eadcomponent_id':
new_component['eadcomponent_id'],
'number_of_scans': {
'set': new_component['number_of_scans'] + 1
}
}])
# get the complete data of the old archive file
old_archivefile = search_archivefiles(context=context,
with_facets=False,
limit=1,
**old_archivefile)['results'][0]
old_archivefile[
'number_of_scans'] = old_archivefile['number_of_scans'] - 1
# we need to manually set the sort_field, as this field is not stored, and therefore not returned
old_archivefile['sort_field'] = sort_field(
archive_id=old_archivefile['archive_id'],
archiveFile=old_archivefile['archiveFile'])
context.solr_archivefile.update([old_archivefile])
# now check for the new archivefile (and add it if necessary)
new_archivefile = cast_scan_as_archivefile(
context, {
'archive_id': scan.archive_id,
'archiveFile': scan.archiveFile
}).get_solr_data(context)
context.solr_archivefile.update([new_archivefile])
if filecontents:
# erase other images
for image in scan.images:
scan.remove_file(image.id)
context.db.delete(image)
context.db.refresh(scan)
image = ScanImage(filename=filename,
scan_number=scan.number,
is_default=True)
scan.images.append(image)
context.db.flush()
scan.store_file(filecontents, image.id)
#
# we need to ping the pagebrowser that the archivefile has changed
#
archivefiles = [ArchiveFile(**old_archivefile)]
if old_archivefile != new_archivefile:
archivefiles.append(ArchiveFile(**new_archivefile))
for archivefile in archivefiles:
for ead_id in archivefile.get_ead_ids(context):
pagebrowser.update.refresh_book(
context,
ead_id=ead_id,
archivefile_id=archivefile.create_archivefile_id())
def add_scan(request):
"""
Add a new scan.
parameters:
%(PARAM_ARCHIVE_ID)s
* **archiveFile:**
identifier of an archive file within the archive
(given by archive_id)
* **file:**
the image of the scan. Must be TIFF, GIF, PNG or JPEG.
* **status:**
a status: a value among :ref:`status_values` (except 0)
* **user:**
the name of a user - optional, will be used for logging info
* **date:**
a date for the scan. If this value is not given, the current date/time will be used.
:other parameters:
all parameters from the data model :ref:`datamodel_scans`
:returns:
information about the scan
See :ref:`TestScans.test_add`
and :ref:`TestScanImages.test_scan_get_has_key_images`.
""" # The last line in the docstring is needed to avoid a sphynx warning
data = prepare_data(request)
scan = Scan()
for key in data:
setattr(scan, key, data[key])
scan.sequenceNumber = find_next_sequence_number(request, scan.archive_id,
scan.archiveFile)
if not scan.date:
scan.date = now()
scan.last_modified = now()
request.db.add(scan)
request.db.flush()
add_files_from_request(request, scan, is_default=True)
request.db.refresh(scan)
request._dbentity = dict(scan=scan)
user = get_user(request)
log_events(request.db, user, [{
'message': 'create',
'object_id': scan.number,
'object_type': 'scan'
}])
# TOOD: optimization: next line can be optimized (using partial_update_keys)
scandata = scan.get_solr_data()
request.solr_scan.update([scandata])
# we need to update the corresponding archive file
document = cast_scan_as_archivefile(request,
scandata).get_solr_data(request)
document['number_of_scans'] = document['number_of_scans'] + 1
def partial_update_keys(document):
del document['title']
del document['titles']
keep_these_original = [
'_version', 'id', 'archive_id', 'archivefile_id'
]
partial_document = dict(
[(k, document[k]) for k in document if k in keep_these_original] +
[(k, {
'set': document[k]
}) for k in document if k not in keep_these_original])
return partial_document
request.solr_archivefile.update([partial_update_keys(document)])
component = cast_archivefile_as_component(request, document)
if component:
try:
request.solr_eadcomponent.update([component])
except SolrException as error:
if error.message['responseHeader']['status'] == 409:
# this probably is a version conflict error [???]
component['_version_'] = 0
request.solr_eadcomponent.update([component])
else:
raise
for ead_id in document['ead_ids']:
pagebrowser.update.refresh_book(
request, ead_id=ead_id, archivefile_id=document['archivefile_id'])
# TODO: refactor: WHY IS THIS NEEDED HERE???
try:
# using request.db.commit on a running server throws an error
# but in tests we seem to need it.
request.db.commit()
except:
pass
return desolarize(scandata, request)