def run_add_image_datastream( data ):
""" Runner for add_image_datastream().
Called by queue-job triggered by tasks.task_manager.determine_next_task(). """
assert sorted( data.keys() ) == ['item_data', 'pid', 'update_image'], sorted( data.keys() )
logger = bell_logger.setup_logger()
ih = ImageHandler( data, logger )
ih.add_image_datastream()
task_manager.determine_next_task( current_task=unicode(sys._getframe().f_code.co_name), logger=logger,
data=data )
return
def run_check_update_image( data ):
""" Runner for check_update_image().
Called by queue-job triggered by tasks.task_manager.determine_next_task(). """
assert sorted( data.keys() ) == ['item_data', 'pid'], sorted( data.keys() )
logger = bell_logger.setup_logger()
ih = ImageHandler( data, logger )
update_image = ih.check_update_image()
task_manager.determine_next_task( current_task=unicode(sys._getframe().f_code.co_name), logger=logger,
data={ 'item_data': data['item_data'], 'pid': data['pid'], 'update_image': update_image } )
return
def delete_jp2( data ):
""" Cleans up created derivative. """
logger = bell_logger.setup_logger()
( item_data_dict, jp2_path, pid ) = ( data['item_data'], data['jp2_path'], data['pid'] )
assert jp2_path[-4:] == '.jp2'
os.remove( jp2_path )
task_manager.determine_next_task(
unicode(sys._getframe().f_code.co_name),
data={ 'item_data': item_data_dict, 'pid': pid },
logger=logger
)
return
def ensure_redis():
""" Checks that redis is running. """
logger = bell_logger.setup_logger()
logger.info( 'STARTING_PROCESSING...' )
try:
assert len(r.keys()) > -1 # if redis isn't running this will generate an error
next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger ) # passes current function name
# job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 )
logger.info( 'in check_environment.ensure_redis(); redis-check ok' )
return
except Exception as e:
message = 'in check_environment.ensure_redis(); redis does not appear to be running; exception: %s' % unicode(repr(e))
logger.error( message )
raise Exception( message )
def archive_previous_work():
""" Archives previous redis data. """
logger = bell_logger.setup_logger()
try:
bell_dir = unicode( os.environ.get('BELL_LOG_DIR') )
now_string = unicode( datetime.datetime.now() ).replace( ' ', '_' )
archive_file_path = '%s/%s.archive' % ( bell_dir, now_string )
jstring = _convert_tracker_to_dict()
with open( archive_file_path, 'w' ) as f:
f.write( jstring )
next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger )
# job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 )
logger.info( 'in check_environment.archive_previous_work(); archive_previous_work ok' )
return
except Exception as e:
message = 'in check_environment.archive_previous_work(); problem archiving previous work; exception: %s' % unicode(repr(e))
logger.error( message )
raise Exception( message )
def check_foundation_files():
""" Checks that foundation-files exist. """
logger = bell_logger.setup_logger()
try:
for filepath in [ os.environ.get('BELL_CE__BELL_DICT_JSON_PATH'), os.environ.get('BELL_CE__AccToPidDict_JSON_PATH') ]:
try:
assert os.path.exists( filepath )
except Exception as e:
message = 'Problem finding filepath %s; exception: %s' % ( filepath, unicode(repr(e)) )
logger.error( message ); raise Exception( message )
task_manager.update_tracker( key='GENERAL', message='foundation files ok' )
next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger )
# job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 )
logger.info( 'in check_environment.check_foundation_files(); files ok' )
return
except Exception as e:
message = 'in check_environment.check_foundation_files(); problem checking foundation files; exception: %s' % unicode(repr(e))
logger.error( message )
raise Exception( message )
def ensure_redis_status_dict():
""" Ensures the status dict exists. Resets it if required.
Each key's value is a json-serializable list. """
logger = bell_logger.setup_logger()
try:
OVERWRITE = unicode( os.environ.get('BELL_CE__TRACKER_OVERWRITE') )
tracker_key = 'bell:tracker'
if OVERWRITE == 'TRUE':
r.delete( tracker_key )
if not r.exists( tracker_key ):
message = '%s initialized %s' % ( tracker_key, unicode(datetime.datetime.now()) )
r.hset( tracker_key, 'GENERAL', json.dumps([message]) )
next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger )
# job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 )
logger.info( 'in check_environment.ensure_redis_status_dict(); bell_status ok' )
return
except Exception as e:
message = 'in check_environment.ensure_redis_status_dict(); redis bell_status not set; exception: %s' % unicode(repr(e))
logger.error( message )
raise Exception( message )
def update_existing_metadata_and_create_image( self,
MASTER_IMAGES_DIR_PATH, MASTER_IMAGES_DIR_URL, JP2_IMAGES_DIR_PATH, JP2_IMAGES_DIR_URL,
pid, item_data_dict, API_URL, logger=None
):
""" CONTROLLER
Note: item_data_dict is a json entry from foundation/acc_num_to_data.py json """
#Store logger if exists
if logger:
self.logger = logger
#Store accession number
self.accession_number = item_data_dict['calc_accession_id'] # for logging
#
#Setup builders
image_builder = ImageBuilder( logger )
#
#Create jp2
master_filename_raw = item_data_dict['object_image_scan_filename'] # includes spaces
master_filename_utf8 = master_filename_raw.encode( 'utf-8' )
master_filename_encoded = urllib.quote( master_filename_utf8 ).decode( 'utf-8' )
source_filepath = '%s/%s' % ( MASTER_IMAGES_DIR_PATH, master_filename_raw )
self.logger.info( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); source_filepath, %s' % source_filepath )
temp_jp2_filename = master_filename_raw.replace( ' ', '_' )
jp2_filename = temp_jp2_filename[0:-4] + '.jp2'
destination_filepath = '%s/%s' % ( JP2_IMAGES_DIR_PATH, jp2_filename )
self.logger.info( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); destination_filepath, %s' % destination_filepath )
image_builder.create_jp2( source_filepath, destination_filepath )
print '- jp2 created.'
#
#Prepare api call
master_url = '%s/%s' % ( MASTER_IMAGES_DIR_URL, master_filename_encoded )
jp2_url = '%s/%s' % ( JP2_IMAGES_DIR_URL, jp2_filename )
params = { 'pid': pid }
params['content_streams'] = json.dumps([
{ 'dsID': 'MASTER', 'url': master_url },
{ 'dsID': 'JP2', 'url': jp2_url }
])
print 'api params prepared'
#
#Make api call
r = requests.put( API_URL, data=params, verify=False )
print 'http put executed'
self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); r.status_code, %s' % r.status_code )
self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); r.content, %s' % r.content.decode('utf-8') )
#
#Read response
resp = r.json()
self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); api_response, %s' % resp )
#
#Update logging
print '- done.'
self._update_task_tracker( message='save_successful' )
print 'response: `%s` logged' % resp
#
#Set next task
task_manager.determine_next_task(
unicode(sys._getframe().f_code.co_name),
data={ 'item_data': item_data_dict, 'jp2_path': destination_filepath, 'pid': pid },
logger=logger
)
print '- next task set.'
# done
return
