logging.info("Setting all tasks to idle.")
tasks = openVDM.get_tasks()
for task in tasks:
openVDM.set_idle_task(task['taskID'])
logging.info("Setting all Collection System Transfers to idle.")
collection_system_transfers = openVDM.get_collection_system_transfers()
for collection_system_transfer in collection_system_transfers:
if not collection_system_transfer['status'] == '3':
openVDM.set_idle_collection_system_transfer(
collection_system_transfer['collectionSystemTransferID'])
logging.info("Setting all Cruise Data Transfers to idle.")
cruise_data_transfers = openVDM.get_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
if not cruise_data_transfer['status'] == '3':
openVDM.set_idle_cruise_data_transfer(
cruise_data_transfer['cruiseDataTransferID'])
required_cruise_data_transfers = openVDM.get_required_cruise_data_transfers(
)
for required_cruise_data_transfer in required_cruise_data_transfers:
if not required_cruise_data_transfer['status'] == '3':
openVDM.set_idle_cruise_data_transfer(
required_cruise_data_transfer['cruiseDataTransferID'])
logging.info("Clearing all jobs from Gearman.")
openVDM.clear_gearman_jobs_from_db()
logging.info("Done!")
cruise_data_transfers = ovdm.get_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
logging.info("Submitting cruise data transfer job for: %s", cruise_data_transfer['longName'])
gmData = {
'cruiseDataTransfer': {
'cruiseDataTransferID': cruise_data_transfer['cruiseDataTransferID']
}
}
completed_job_request = gm_client.submit_job("runCruiseDataTransfer", json.dumps(gmData), background=True)
time.sleep(2)
required_cruise_data_transfers = ovdm.get_required_cruise_data_transfers()
for required_cruise_data_transfer in required_cruise_data_transfers:
if required_cruise_data_transfer['name'] == 'SSDW':
logging.info("Submitting cruise data transfer job for: %s", required_cruise_data_transfer['longName'])
gmData = {
}
completed_job_request = gm_client.submit_job("runShipToShoreTransfer", json.dumps(gmData), background=True)
time.sleep(2)
delay = parsed_args.interval * 60 - len(collection_system_transfers) * 2 - len(cruise_data_transfers) * 2 - 2
logging.info("Waiting %s seconds until next round of tasks are queued", delay)
time.sleep(delay)
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.job_pid = ''
self.job_info = {}
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def _get_job_info(self):
"""
Fetch job metadata
"""
collection_system_transfers = self.ovdm.get_collection_system_transfers(
)
for collection_system_transfer in collection_system_transfers:
if collection_system_transfer['pid'] == self.job_pid:
return {
'type': 'collectionSystemTransfer',
'id':
collection_system_transfer['collectionSystemTransferID'],
'name': collection_system_transfer['name'],
'pid': collection_system_transfer['pid']
}
cruise_data_transfers = self.ovdm.get_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
if cruise_data_transfer['pid'] != "0":
return {
'type': 'cruiseDataTransfer',
'id': cruise_data_transfer['cruiseDataTransferID'],
'name': cruise_data_transfer['name'],
'pid': cruise_data_transfer['pid']
}
cruise_data_transfers = self.ovdm.get_required_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
if cruise_data_transfer['pid'] != "0":
return {
'type': 'cruiseDataTransfer',
'id': cruise_data_transfer['cruiseDataTransferID'],
'name': cruise_data_transfer['name'],
'pid': cruise_data_transfer['pid']
}
tasks = self.ovdm.get_tasks()
for task in tasks:
if task['pid'] != "0":
return {
'type': 'task',
'id': task['taskID'],
'name': task['name'],
'pid': task['pid']
}
return {'type': 'unknown'}
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
payload_obj = json.loads(current_job.data)
self.job_pid = payload_obj['pid']
self.job_info = self._get_job_info()
logging.info("Job: %s, Killing PID: %s failed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.info("Job: %s Killing PID %s failed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker Crashed",
"result": "Fail",
"reason": "Unknown"
}]))
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, Killing PID %s completed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.system_status = self.ovdm.get_system_status()
self.transfer_start_date = None
self.cruise_data_transfer = self._get_cruise_data_transfer()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def _get_cruise_data_transfer(self):
"""
Fetch the ship-to-shore transfer configuration
"""
try:
return list(
filter(lambda transfer: transfer['name'] == 'SSDW',
self.ovdm.get_required_cruise_data_transfers()))[0]
except Exception as err:
logging.error("Could not find SSDW transfer configuration")
logging.debug(str(err))
return None
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
self.stop = False
payload_obj = json.loads(current_job.data)
self.cruise_data_transfer = self._get_cruise_data_transfer()
if not self.cruise_data_transfer:
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer Found",
"result": "Fail",
"reason": "Transfer configuration not found"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
if 'cruiseDataTransfer' in payload_obj:
self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])
self.system_status = payload_obj[
'systemStatus'] if 'systemStatus' in payload_obj else self.ovdm.get_system_status(
)
if self.system_status == "Off" or self.cruise_data_transfer[
'enable'] == '0':
logging.info(
"Ship-to-shore Transfer job skipped because ship-to-shore transfers are currently disabled"
)
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer Enabled",
"result": "Ignore",
"reason": "Transfer is disabled"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
bw_limit_status = payload_obj[
'bandwidthLimitStatus'] if 'bandwidthLimitStatus' in payload_obj else self.ovdm.get_ship_to_shore_bw_limit_status(
)
if not bw_limit_status:
self.cruise_data_transfer['bandwidthLimit'] = '0'
self.cruise_id = payload_obj[
'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
)
self.transfer_start_date = time.strftime("%Y%m%dT%H%M%SZ",
time.gmtime())
logging.info("Job: %s, Ship-to-Shore transfer started at: %s",
current_job.handle, time.strftime("%D %T", time.gmtime()))
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s, %s transfer failed at: %s", current_job.handle,
self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker crashed",
"result": "Fail",
"reason": "Unknown"
}]))
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
results_obj['parts'][-1]['reason'])
elif results_obj['parts'][-1]['result'] == "Pass":
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
else:
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, %s transfer completed at: %s",
current_job.handle, self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
