class OVDMGearmanWorker(python3_gearman.GearmanWorker): # pylint: disable=too-many-instance-attributes """ Class for the current Gearman worker """ def __init__(self): self.stop = False self.ovdm = OpenVDM() self.task = None self.files = {'new': [], 'updated': []} self.cruise_id = self.ovdm.get_cruise_id() self.lowering_id = self.ovdm.get_lowering_id() self.collection_system_transfer = {'name': ""} self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config( ) super().__init__(host_list=[self.ovdm.get_gearman_server()]) @staticmethod def _get_custom_task(current_job): """ Fetch task metadata """ task = list( filter(lambda task: task['name'] == current_job.task, CUSTOM_TASKS)) return task[0] if len(task) > 0 else None def on_job_execute(self, current_job): """ Function run whenever a new job arrives """ logging.debug("current_job: %s", current_job) self.stop = False payload_obj = json.loads(current_job.data) self.task = self._get_custom_task( current_job) if self._get_custom_task( current_job) is not None else self.ovdm.get_task_by_name( current_job.task) logging.debug("task: %s", self.task) if self.task is None: self.on_job_complete( current_job, json.dumps({ 'parts': [{ "partName": "Verify Task", "result": "Fail", "reason": "undefined task" }] })) elif int(self.task['taskID']) > 0: self.ovdm.set_running_task(self.task['taskID'], os.getpid(), current_job.handle) else: self.ovdm.track_gearman_job(self.task['longName'], os.getpid(), current_job.handle) logging.info("Job: %s (%s) started at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) self.files = payload_obj['files'] if 'files' in payload_obj else { 'new': [], 'updated': [] } self.cruise_id = payload_obj[ 'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id( ) self.lowering_id = payload_obj[ 'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id( ) self.collection_system_transfer = self.ovdm.get_collection_system_transfer( payload_obj['collectionSystemTransferID'] ) if 'collectionSystemTransferID' in payload_obj else { 'name': "" } self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config( ) return super().on_job_execute(current_job) def on_job_exception(self, current_job, exc_info): """ Function run whenever the current job has an exception """ logging.error("Job: %s (%s) failed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) self.send_job_data( current_job, json.dumps([{ "partName": "Worker crashed", "result": "Fail", "reason": "Unknown" }])) if int(self.task['taskID']) > 0: self.ovdm.set_error_task(self.task['taskID'], "Worker crashed") else: self.ovdm.send_msg(self.task['longName'] + ' failed', 'Worker crashed') exc_type, _, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.error(exc_type, fname, exc_tb.tb_lineno) return super().on_job_exception(current_job, exc_info) def on_job_complete(self, current_job, job_result): """ Function run whenever the current job completes """ results_obj = json.loads(job_result) if len(results_obj['parts']) > 0: if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict if int(self.task['taskID']) > 0: self.ovdm.set_error_task( self.task['taskID'], results_obj['parts'][-1]['reason']) else: self.ovdm.send_msg(self.task['longName'] + ' failed', results_obj['parts'][-1]['reason']) else: if int(self.task['taskID']) > 0: self.ovdm.set_idle_task(self.task['taskID']) else: if int(self.task['taskID']) > 0: self.ovdm.set_idle_task(self.task['taskID']) logging.debug("Job Results: %s", json.dumps(results_obj, indent=2)) logging.info("Job: %s (%s) completed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) return super().send_job_complete(current_job, job_result) def stop_task(self): """ Function to stop the current job """ self.stop = True logging.warning("Stopping current task...") def quit_worker(self): """ Function to quit the worker """ self.stop = True logging.warning("Quitting worker...") self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker): # pylint: disable=too-many-instance-attributes """ Class for the current Gearman worker """ def __init__(self): self.stop = False self.ovdm = OpenVDM() self.task = None self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config( ) self.cruise_id = self.ovdm.get_cruise_id() self.cruise_dir = os.path.join( self. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id) self.lowering_id = self.ovdm.get_cruise_id() self.lowering_dir = os.path.join( self.cruise_dir, self.shipboard_data_warehouse_config['loweringDataBaseDir'], self.lowering_id) if self.lowering_id else None self.data_dashboard_dir = os.path.join( self.cruise_dir, self.ovdm.get_required_extra_directory_by_name('Dashboard_Data') ['destDir']) self.data_dashboard_manifest_file_path = os.path.join( self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN) self.collection_system_transfer = {} super().__init__(host_list=[self.ovdm.get_gearman_server()]) @staticmethod def get_custom_task(current_job): """ Retrieve task metadata """ task = list( filter(lambda task: task['name'] == current_job.task, customTasks)) return task[0] if len(task) > 0 else None def on_job_execute(self, current_job): """ Function run whenever a new job arrives """ logging.debug("current_job: %s", current_job) payload_obj = json.loads(current_job.data) self.task = self.get_custom_task(current_job) if self.get_custom_task( current_job) is not None else self.ovdm.get_task_by_name( current_job.task) logging.debug("task: %s", self.task) if int(self.task['taskID']) > 0: self.ovdm.set_running_task(self.task['taskID'], os.getpid(), current_job.handle) else: self.ovdm.track_gearman_job(self.task['longName'], os.getpid(), current_job.handle) logging.info("Job: %s (%s) started at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config( ) self.cruise_id = payload_obj[ 'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id( ) self.cruise_dir = os.path.join( self. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id) self.lowering_id = payload_obj[ 'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id( ) self.lowering_dir = os.path.join( self.cruise_dir, self.shipboard_data_warehouse_config['loweringDataBaseDir'], self.lowering_id) if self.lowering_id else None self.collection_system_transfer = self.ovdm.get_collection_system_transfer( payload_obj['collectionSystemTransferID'] ) if 'collectionSystemTransferID' in payload_obj else { 'name': "Unknown" } self.data_dashboard_dir = os.path.join( self.cruise_dir, self.ovdm.get_required_extra_directory_by_name('Dashboard_Data') ['destDir']) self.data_dashboard_manifest_file_path = os.path.join( self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN) return super().on_job_execute(current_job) def on_job_exception(self, current_job, exc_info): """ Function run whenever the current job has an exception """ logging.error("Job: %s (%s) failed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) self.send_job_data( current_job, json.dumps([{ "partName": "Worker crashed", "result": "Fail", "reason": "Unknown, contact Webb :-)" }])) if int(self.task['taskID']) > 0: self.ovdm.set_error_task(self.task['taskID'], "Worker crashed") else: self.ovdm.send_msg(self.task['longName'] + ' failed', 'Worker crashed') exc_type, _, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.error(exc_type, fname, exc_tb.tb_lineno) return super().on_job_exception(current_job, exc_info) def on_job_complete(self, current_job, job_result): """ Function run whenever the current job completes """ results_obj = json.loads(job_result) logging.debug("Preparing subsequent Gearman jobs") job_data = { 'cruiseID': self.cruise_id, 'loweringID': self.lowering_id, 'files': results_obj['files'] } if current_job.task == 'updateDataDashboard': gm_client = python3_gearman.GearmanClient( [self.ovdm.get_gearman_server()]) payload_obj = json.loads(current_job.data) job_data['collectionSystemTransferID'] = payload_obj[ 'collectionSystemTransferID'] for task in self.ovdm.get_tasks_for_hook(current_job.task): logging.info("Adding post task: %s", task) gm_client.submit_job(task, json.dumps(job_data), background=True) elif current_job.task == 'rebuildDataDashboard': gm_client = python3_gearman.GearmanClient( [self.ovdm.get_gearman_server()]) for task in self.ovdm.get_tasks_for_hook(current_job.task): logging.info("Adding post task: %s", task) gm_client.submit_job(task, json.dumps(job_data), background=True) if len(results_obj['parts']) > 0: if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict if int(self.task['taskID']) > 0: self.ovdm.set_error_task( self.task['taskID'], results_obj['parts'][-1]['reason']) else: self.ovdm.send_msg(self.task['longName'] + ' failed', results_obj['parts'][-1]['reason']) else: if int(self.task['taskID']) > 0: self.ovdm.set_idle_task(self.task['taskID']) else: if int(self.task['taskID']) > 0: self.ovdm.set_idle_task(self.task['taskID']) logging.debug("Job Results: %s", json.dumps(results_obj, indent=2)) logging.info("Job: %s (%s) completed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime())) return super().send_job_complete(current_job, job_result) def stop_task(self): """ Function to stop the current job """ self.stop = True logging.warning("Stopping current task...") def quit_worker(self): """ Function to quit the worker """ self.stop = True logging.warning("Quitting worker...") self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker): """ Class for the current Gearman worker """ def __init__(self): self.stop = False self.ovdm = OpenVDM() self.cruise_id = self.ovdm.get_cruise_id() self.lowering_id = self.ovdm.get_lowering_id() self.collection_system_transfer = {} self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config() super().__init__(host_list=[self.ovdm.get_gearman_server()]) def on_job_execute(self, current_job): """ Function run whenever a new job arrives """ logging.debug("current_job: %s", current_job) self.stop = False payload_obj = json.loads(current_job.data) if 'collectionSystemTransferID' in payload_obj['collectionSystemTransfer']: self.collection_system_transfer = self.ovdm.get_collection_system_transfer(payload_obj['collectionSystemTransfer']['collectionSystemTransferID']) if not self.collection_system_transfer: return self.on_job_complete(current_job, json.dumps({'parts':[{"partName": "Located Collection System Tranfer Data", "result": "Fail", "reason": "Could not find configuration data for collection system transfer"},{"partName": "Final Verdict", "result": "Fail", "reason": "Could not find configuration data for collection system transfer"}]})) self.collection_system_transfer.update(payload_obj['collectionSystemTransfer']) else: self.collection_system_transfer = payload_obj['collectionSystemTransfer'] self.cruise_id = payload_obj['cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id() self.lowering_id = payload_obj['loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id() if self.collection_system_transfer['cruiseOrLowering'] == '1' and self.lowering_id is None: try: return self.on_job_complete(current_job, json.dumps({'parts':[{"partName": "Retrieve Lowering ID", "result": "Fail", "reason": "Lowering ID is not defined"},{"partName": "Final Verdict", "result": "Fail", "reason": "Lowering ID is not defined"}]})) except Exception as err: raise err logging.info("Job: %s, %s transfer test started at: %s", current_job.handle, self.collection_system_transfer['name'], time.strftime("%D %T", time.gmtime())) self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config() return super().on_job_execute(current_job) def on_job_exception(self, current_job, exc_info): """ Function run whenever the current job has an exception """ logging.error("Job: %s, %s transfer test failed at: %s", current_job.handle, self.collection_system_transfer['name'], time.strftime("%D %T", time.gmtime())) self.send_job_data(current_job, json.dumps([{"partName": "Worker crashed", "result": "Fail", "reason": "Unknown"}])) if 'collectionSystemTransferID' in self.collection_system_transfer: self.ovdm.set_error_collection_system_transfer_test(self.collection_system_transfer['collectionSystemTransferID'], 'Worker crashed') exc_type, _, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.error(exc_type, fname, exc_tb.tb_lineno) return super().on_job_exception(current_job, exc_info) def on_job_complete(self, current_job, job_result): """ Function run whenever the current job completes """ results_obj = json.loads(job_result) if 'collectionSystemTransferID' in self.collection_system_transfer: if len(results_obj['parts']) > 0: if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict self.ovdm.set_error_collection_system_transfer_test(self.collection_system_transfer['collectionSystemTransferID'], results_obj['parts'][-1]['reason']) else: self.ovdm.clear_error_collection_system_transfer(self.collection_system_transfer['collectionSystemTransferID'], self.collection_system_transfer['status']) else: self.ovdm.clear_error_collection_system_transfer(self.collection_system_transfer['collectionSystemTransferID'], self.collection_system_transfer['status']) logging.debug("Job Results: %s", json.dumps(results_obj, indent=2)) logging.info("Job: %s, %s transfer test completed at: %s", current_job.handle, self.collection_system_transfer['name'], time.strftime("%D %T", time.gmtime())) return super().send_job_complete(current_job, job_result) def stop_task(self): """ Function to stop the current job """ self.stop = True logging.warning("Stopping current task...") def quit_worker(self): """ Function to quit the worker """ self.stop = True logging.warning("Quitting worker...") self.shutdown()