def run_add_image_datastream( data ): """ Runner for add_image_datastream(). Called by queue-job triggered by tasks.task_manager.determine_next_task(). """ assert sorted( data.keys() ) == ['item_data', 'pid', 'update_image'], sorted( data.keys() ) logger = bell_logger.setup_logger() ih = ImageHandler( data, logger ) ih.add_image_datastream() task_manager.determine_next_task( current_task=unicode(sys._getframe().f_code.co_name), logger=logger, data=data ) return
def run_check_update_image( data ): """ Runner for check_update_image(). Called by queue-job triggered by tasks.task_manager.determine_next_task(). """ assert sorted( data.keys() ) == ['item_data', 'pid'], sorted( data.keys() ) logger = bell_logger.setup_logger() ih = ImageHandler( data, logger ) update_image = ih.check_update_image() task_manager.determine_next_task( current_task=unicode(sys._getframe().f_code.co_name), logger=logger, data={ 'item_data': data['item_data'], 'pid': data['pid'], 'update_image': update_image } ) return
def delete_jp2( data ): """ Cleans up created derivative. """ logger = bell_logger.setup_logger() ( item_data_dict, jp2_path, pid ) = ( data['item_data'], data['jp2_path'], data['pid'] ) assert jp2_path[-4:] == '.jp2' os.remove( jp2_path ) task_manager.determine_next_task( unicode(sys._getframe().f_code.co_name), data={ 'item_data': item_data_dict, 'pid': pid }, logger=logger ) return
def ensure_redis(): """ Checks that redis is running. """ logger = bell_logger.setup_logger() logger.info( 'STARTING_PROCESSING...' ) try: assert len(r.keys()) > -1 # if redis isn't running this will generate an error next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger ) # passes current function name # job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 ) logger.info( 'in check_environment.ensure_redis(); redis-check ok' ) return except Exception as e: message = 'in check_environment.ensure_redis(); redis does not appear to be running; exception: %s' % unicode(repr(e)) logger.error( message ) raise Exception( message )
def archive_previous_work(): """ Archives previous redis data. """ logger = bell_logger.setup_logger() try: bell_dir = unicode( os.environ.get('BELL_LOG_DIR') ) now_string = unicode( datetime.datetime.now() ).replace( ' ', '_' ) archive_file_path = '%s/%s.archive' % ( bell_dir, now_string ) jstring = _convert_tracker_to_dict() with open( archive_file_path, 'w' ) as f: f.write( jstring ) next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger ) # job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 ) logger.info( 'in check_environment.archive_previous_work(); archive_previous_work ok' ) return except Exception as e: message = 'in check_environment.archive_previous_work(); problem archiving previous work; exception: %s' % unicode(repr(e)) logger.error( message ) raise Exception( message )
def check_foundation_files(): """ Checks that foundation-files exist. """ logger = bell_logger.setup_logger() try: for filepath in [ os.environ.get('BELL_CE__BELL_DICT_JSON_PATH'), os.environ.get('BELL_CE__AccToPidDict_JSON_PATH') ]: try: assert os.path.exists( filepath ) except Exception as e: message = 'Problem finding filepath %s; exception: %s' % ( filepath, unicode(repr(e)) ) logger.error( message ); raise Exception( message ) task_manager.update_tracker( key='GENERAL', message='foundation files ok' ) next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger ) # job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 ) logger.info( 'in check_environment.check_foundation_files(); files ok' ) return except Exception as e: message = 'in check_environment.check_foundation_files(); problem checking foundation files; exception: %s' % unicode(repr(e)) logger.error( message ) raise Exception( message )
def ensure_redis_status_dict(): """ Ensures the status dict exists. Resets it if required. Each key's value is a json-serializable list. """ logger = bell_logger.setup_logger() try: OVERWRITE = unicode( os.environ.get('BELL_CE__TRACKER_OVERWRITE') ) tracker_key = 'bell:tracker' if OVERWRITE == 'TRUE': r.delete( tracker_key ) if not r.exists( tracker_key ): message = '%s initialized %s' % ( tracker_key, unicode(datetime.datetime.now()) ) r.hset( tracker_key, 'GENERAL', json.dumps([message]) ) next = task_manager.determine_next_task( sys._getframe().f_code.co_name, logger=logger ) # job = q.enqueue_call ( func='%s' % next, args=(), timeout=30 ) logger.info( 'in check_environment.ensure_redis_status_dict(); bell_status ok' ) return except Exception as e: message = 'in check_environment.ensure_redis_status_dict(); redis bell_status not set; exception: %s' % unicode(repr(e)) logger.error( message ) raise Exception( message )
def update_existing_metadata_and_create_image( self, MASTER_IMAGES_DIR_PATH, MASTER_IMAGES_DIR_URL, JP2_IMAGES_DIR_PATH, JP2_IMAGES_DIR_URL, pid, item_data_dict, API_URL, logger=None ): """ CONTROLLER Note: item_data_dict is a json entry from foundation/acc_num_to_data.py json """ #Store logger if exists if logger: self.logger = logger #Store accession number self.accession_number = item_data_dict['calc_accession_id'] # for logging # #Setup builders image_builder = ImageBuilder( logger ) # #Create jp2 master_filename_raw = item_data_dict['object_image_scan_filename'] # includes spaces master_filename_utf8 = master_filename_raw.encode( 'utf-8' ) master_filename_encoded = urllib.quote( master_filename_utf8 ).decode( 'utf-8' ) source_filepath = '%s/%s' % ( MASTER_IMAGES_DIR_PATH, master_filename_raw ) self.logger.info( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); source_filepath, %s' % source_filepath ) temp_jp2_filename = master_filename_raw.replace( ' ', '_' ) jp2_filename = temp_jp2_filename[0:-4] + '.jp2' destination_filepath = '%s/%s' % ( JP2_IMAGES_DIR_PATH, jp2_filename ) self.logger.info( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); destination_filepath, %s' % destination_filepath ) image_builder.create_jp2( source_filepath, destination_filepath ) print '- jp2 created.' # #Prepare api call master_url = '%s/%s' % ( MASTER_IMAGES_DIR_URL, master_filename_encoded ) jp2_url = '%s/%s' % ( JP2_IMAGES_DIR_URL, jp2_filename ) params = { 'pid': pid } params['content_streams'] = json.dumps([ { 'dsID': 'MASTER', 'url': master_url }, { 'dsID': 'JP2', 'url': jp2_url } ]) print 'api params prepared' # #Make api call r = requests.put( API_URL, data=params, verify=False ) print 'http put executed' self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); r.status_code, %s' % r.status_code ) self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); r.content, %s' % r.content.decode('utf-8') ) # #Read response resp = r.json() self.logger.debug( 'in fedora_metadata_updater_and_image_builder.update_existing_metadata_and_create_image(); api_response, %s' % resp ) # #Update logging print '- done.' self._update_task_tracker( message='save_successful' ) print 'response: `%s` logged' % resp # #Set next task task_manager.determine_next_task( unicode(sys._getframe().f_code.co_name), data={ 'item_data': item_data_dict, 'jp2_path': destination_filepath, 'pid': pid }, logger=logger ) print '- next task set.' # done return