def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_data_transfer = {}
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_dir = os.path.join(
self.
shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
self.cruise_id)
self.lowering_id = self.ovdm.get_cruise_id()
self.lowering_dir = os.path.join(
self.cruise_dir,
self.shipboard_data_warehouse_config['loweringDataBaseDir'],
self.lowering_id) if self.lowering_id else None
self.data_dashboard_dir = os.path.join(
self.cruise_dir,
self.ovdm.get_required_extra_directory_by_name('Dashboard_Data')
['destDir'])
self.data_dashboard_manifest_file_path = os.path.join(
self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN)
self.collection_system_transfer = {}
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_start_date = self.ovdm.get_cruise_start_date()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.system_status = self.ovdm.get_system_status()
self.transfer_start_date = None
self.cruise_data_transfer = self._get_cruise_data_transfer()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.cruise_id = self.ovdm.get_cruise_id()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)
self.md5_summary_filepath = os.path.join(self.cruise_dir, DEFAULT_MD5_SUMMARY_FN)
self.md5_summary_md5_filepath = os.path.join(self.cruise_dir, DEFAULT_MD5_SUMMARY_MD5_FN)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.files = { 'new':[], 'updated':[] }
self.cruise_id = self.ovdm.get_cruise_id()
self.lowering_id = self.ovdm.get_lowering_id()
self.collection_system_transfer = { 'name': "" }
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.cruise_id = self.ovdm.get_cruise_id()
self.lowering_id = self.ovdm.get_lowering_id()
self.lowering_start_date = self.ovdm.get_lowering_start_date()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.lowering_dir = None
super().__init__(host_list=[self.ovdm.get_gearman_server()])
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.system_status = self.ovdm.get_system_status()
self.cruise_data_transfer = {}
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
self.stop = False
payload_obj = json.loads(current_job.data)
try:
self.cruise_data_transfer = self.ovdm.get_cruise_data_transfer(
payload_obj['cruiseDataTransfer']['cruiseDataTransferID'])
if not self.cruise_data_transfer:
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName":
"Located Cruise Data Tranfer Data",
"result":
"Fail",
"reason":
"Could not find configuration data for cruise data transfer"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
if self.cruise_data_transfer['status'] == "1": #not running
logging.info(
"Transfer job for %s skipped because a transfer for that cruise data destination is already in-progress",
self.cruise_data_transfer['name'])
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer In-Progress",
"result": "Fail",
"reason": "Transfer is already in-progress"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
except Exception as err:
logging.debug(str(err))
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName":
"Located Cruise Data Tranfer Data",
"result":
"Fail",
"reason":
"Could not find retrieve data for cruise data transfer from OpenVDM API"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
self.system_status = payload_obj[
'systemStatus'] if 'systemStatus' in payload_obj else self.ovdm.get_system_status(
)
self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])
if self.system_status == "Off" or self.cruise_data_transfer[
'enable'] == '0':
logging.info(
"Transfer job for %s skipped because that cruise data transfer is currently disabled",
self.cruise_data_transfer['name'])
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer Enabled",
"result": "Ignore",
"reason": "Transfer is disabled"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
self.cruise_id = self.ovdm.get_cruise_id()
logging.info("Job: %s, %s transfer started at: %s", current_job.handle,
self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s, %s transfer failed at: %s", current_job.handle,
self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker crashed",
"result": "Fail",
"reason": "Unknown"
}]))
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
results_obj['parts'][-1]['reason'])
else:
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
else:
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, %s transfer completed at: %s",
current_job.handle, self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker): # pylint: disable=too-many-instance-attributes
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.cruise_id = self.ovdm.get_cruise_id()
self.lowering_id = self.ovdm.get_lowering_id()
self.lowering_start_date = self.ovdm.get_lowering_start_date()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.lowering_dir = os.path.join(
self.
shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
self.cruise_id,
self.shipboard_data_warehouse_config['loweringDataBaseDir'],
self.lowering_id)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
@staticmethod
def _get_custom_task(current_job):
"""
Fetch task metadata
"""
task = list(
filter(lambda task: task['name'] == current_job.task,
CUSTOM_TASKS))
return task[0] if len(task) > 0 else None
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
payload_obj = json.loads(current_job.data)
self.task = self._get_custom_task(
current_job) if self._get_custom_task(
current_job) is not None else self.ovdm.get_task_by_name(
current_job.task)
logging.debug("task: %s", self.task)
if int(self.task['taskID']) > 0:
self.ovdm.set_running_task(self.task['taskID'], os.getpid(),
current_job.handle)
# else:
# self.ovdm.track_gearman_job(taskLookup[current_job.task], os.getpid(), current_job.handle)
logging.info("Job: %s (%s) started at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
self.cruise_id = payload_obj[
'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
)
self.lowering_id = payload_obj[
'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id(
)
self.lowering_start_date = payload_obj[
'loweringStartDate'] if 'loweringStartDate' in payload_obj else self.ovdm.get_lowering_start_date(
)
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.lowering_dir = os.path.join(
self.
shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
self.cruise_id,
self.shipboard_data_warehouse_config['loweringDataBaseDir'],
self.lowering_id)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s (%s) failed at: %s",
self.task['longName'], current_job.handle,
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker crashed",
"result": "Fail",
"reason": "Unknown"
}]))
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(
self.task['taskID'],
results_obj['parts'][-1]['reason'])
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
results_obj['parts'][-1]['reason'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
help='Increase output verbosity')
parsed_args = parser.parse_args()
############################
# Set up logging before we do any other argument parsing (so that we
# can log problems with argument parsing).
LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity])
ovdm = OpenVDM()
while True:
start_t = datetime.datetime.utcnow()
warehouse_config = ovdm.get_shipboard_data_warehouse_config()
cruise_dir = join(warehouse_config['shipboardDataWarehouseBaseDir'], ovdm.get_cruise_id())
lowering_id = ovdm.get_lowering_id() if ovdm.get_show_lowering_components() else None
lowering_dir = join(cruise_dir, warehouse_config['loweringDataBaseDir'], lowering_id) if lowering_id else None
logging.debug("Cruise Directory: %s", cruise_dir)
logging.debug("Lowering Directory: %s", lowering_dir)
if isdir(cruise_dir):
help='Increase output verbosity')
parsed_args = parser.parse_args()
############################
# Set up logging before we do any other argument parsing (so that we
# can log problems with argument parsing).
LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity])
openVDM = OpenVDM()
time.sleep(5)
logging.info("Setting all tasks to idle.")
tasks = openVDM.get_tasks()
for task in tasks:
openVDM.set_idle_task(task['taskID'])
logging.info("Setting all Collection System Transfers to idle.")
collection_system_transfers = openVDM.get_collection_system_transfers()
for collection_system_transfer in collection_system_transfers:
if not collection_system_transfer['status'] == '3':
openVDM.set_idle_collection_system_transfer(
collection_system_transfer['collectionSystemTransferID'])
import argparse
import json
import logging
import sys
import time
from os.path import dirname, realpath
from python3_gearman import GearmanClient
sys.path.append(dirname(dirname(dirname(realpath(__file__)))))
from server.lib.openvdm import OpenVDM
if __name__ == "__main__":
ovdm = OpenVDM()
parser = argparse.ArgumentParser(
description='OpenVDM Data Transfer Scheduler')
parser.add_argument('-i',
'--interval',
default=ovdm.get_transfer_interval(),
metavar='interval',
type=int,
help='interval in minutes')
parser.add_argument('-v',
'--verbosity',
dest='verbosity',
default=0,
action='count',
help='Increase output verbosity')
class OVDMGearmanWorker(python3_gearman.GearmanWorker): # pylint: disable=too-many-instance-attributes
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_dir = os.path.join(
self.
shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
self.cruise_id)
self.lowering_id = self.ovdm.get_cruise_id()
self.lowering_dir = os.path.join(
self.cruise_dir,
self.shipboard_data_warehouse_config['loweringDataBaseDir'],
self.lowering_id) if self.lowering_id else None
self.data_dashboard_dir = os.path.join(
self.cruise_dir,
self.ovdm.get_required_extra_directory_by_name('Dashboard_Data')
['destDir'])
self.data_dashboard_manifest_file_path = os.path.join(
self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN)
self.collection_system_transfer = {}
super().__init__(host_list=[self.ovdm.get_gearman_server()])
@staticmethod
def get_custom_task(current_job):
"""
Retrieve task metadata
"""
task = list(
filter(lambda task: task['name'] == current_job.task, customTasks))
return task[0] if len(task) > 0 else None
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
payload_obj = json.loads(current_job.data)
self.task = self.get_custom_task(current_job) if self.get_custom_task(
current_job) is not None else self.ovdm.get_task_by_name(
current_job.task)
logging.debug("task: %s", self.task)
if int(self.task['taskID']) > 0:
self.ovdm.set_running_task(self.task['taskID'], os.getpid(),
current_job.handle)
else:
self.ovdm.track_gearman_job(self.task['longName'], os.getpid(),
current_job.handle)
logging.info("Job: %s (%s) started at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
self.cruise_id = payload_obj[
'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
)
self.cruise_dir = os.path.join(
self.
shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
self.cruise_id)
self.lowering_id = payload_obj[
'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id(
)
self.lowering_dir = os.path.join(
self.cruise_dir,
self.shipboard_data_warehouse_config['loweringDataBaseDir'],
self.lowering_id) if self.lowering_id else None
self.collection_system_transfer = self.ovdm.get_collection_system_transfer(
payload_obj['collectionSystemTransferID']
) if 'collectionSystemTransferID' in payload_obj else {
'name': "Unknown"
}
self.data_dashboard_dir = os.path.join(
self.cruise_dir,
self.ovdm.get_required_extra_directory_by_name('Dashboard_Data')
['destDir'])
self.data_dashboard_manifest_file_path = os.path.join(
self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s (%s) failed at: %s",
self.task['longName'], current_job.handle,
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker crashed",
"result": "Fail",
"reason": "Unknown, contact Webb :-)"
}]))
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
logging.debug("Preparing subsequent Gearman jobs")
job_data = {
'cruiseID': self.cruise_id,
'loweringID': self.lowering_id,
'files': results_obj['files']
}
if current_job.task == 'updateDataDashboard':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
payload_obj = json.loads(current_job.data)
job_data['collectionSystemTransferID'] = payload_obj[
'collectionSystemTransferID']
for task in self.ovdm.get_tasks_for_hook(current_job.task):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
elif current_job.task == 'rebuildDataDashboard':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook(current_job.task):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(
self.task['taskID'],
results_obj['parts'][-1]['reason'])
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
results_obj['parts'][-1]['reason'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.system_status = self.ovdm.get_system_status()
self.transfer_start_date = None
self.cruise_data_transfer = self._get_cruise_data_transfer()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def _get_cruise_data_transfer(self):
"""
Fetch the ship-to-shore transfer configuration
"""
try:
return list(
filter(lambda transfer: transfer['name'] == 'SSDW',
self.ovdm.get_required_cruise_data_transfers()))[0]
except Exception as err:
logging.error("Could not find SSDW transfer configuration")
logging.debug(str(err))
return None
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
self.stop = False
payload_obj = json.loads(current_job.data)
self.cruise_data_transfer = self._get_cruise_data_transfer()
if not self.cruise_data_transfer:
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer Found",
"result": "Fail",
"reason": "Transfer configuration not found"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
if 'cruiseDataTransfer' in payload_obj:
self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])
self.system_status = payload_obj[
'systemStatus'] if 'systemStatus' in payload_obj else self.ovdm.get_system_status(
)
if self.system_status == "Off" or self.cruise_data_transfer[
'enable'] == '0':
logging.info(
"Ship-to-shore Transfer job skipped because ship-to-shore transfers are currently disabled"
)
return self.on_job_complete(
current_job,
json.dumps({
'parts': [{
"partName": "Transfer Enabled",
"result": "Ignore",
"reason": "Transfer is disabled"
}],
'files': {
'new': [],
'updated': [],
'exclude': []
}
}))
bw_limit_status = payload_obj[
'bandwidthLimitStatus'] if 'bandwidthLimitStatus' in payload_obj else self.ovdm.get_ship_to_shore_bw_limit_status(
)
if not bw_limit_status:
self.cruise_data_transfer['bandwidthLimit'] = '0'
self.cruise_id = payload_obj[
'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
)
self.transfer_start_date = time.strftime("%Y%m%dT%H%M%SZ",
time.gmtime())
logging.info("Job: %s, Ship-to-Shore transfer started at: %s",
current_job.handle, time.strftime("%D %T", time.gmtime()))
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s, %s transfer failed at: %s", current_job.handle,
self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker crashed",
"result": "Fail",
"reason": "Unknown"
}]))
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
self.ovdm.set_error_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'],
results_obj['parts'][-1]['reason'])
elif results_obj['parts'][-1]['result'] == "Pass":
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
else:
self.ovdm.set_idle_cruise_data_transfer(
self.cruise_data_transfer['cruiseDataTransferID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, %s transfer completed at: %s",
current_job.handle, self.cruise_data_transfer['name'],
time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.task = None
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_start_date = self.ovdm.get_cruise_start_date()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
payload_obj = json.loads(current_job.data)
self.stop = False
self.task = self.ovdm.get_task_by_name(current_job.task)
logging.debug("task: %s", self.task)
if int(self.task['taskID']) > 0:
self.ovdm.set_running_task(self.task['taskID'], os.getpid(), current_job.handle)
# else:
# self.ovdm.track_gearman_job(task_lookup[current_job.task], os.getpid(), current_job.handle)
logging.info("Job: %s (%s) started at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))
self.cruise_id = payload_obj['cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id()
self.cruise_start_date = payload_obj['cruiseStartDate'] if 'cruiseStartDate' in payload_obj else self.ovdm.get_cruise_start_date()
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s (%s) failed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))
self.send_job_data(current_job, json.dumps([{"partName": "Worker crashed", "result": "Fail", "reason": "Worker crashed"}]))
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
else:
self.ovdm.send_msg(self.task['longName'] + ' failed', 'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
job_data = {
'cruiseID': self.cruise_id,
'cruiseStartDate': self.cruise_start_date
}
if current_job.task == "setupNewCruise":
gm_client = python3_gearman.GearmanClient([self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook('setupNewCruise'):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task, json.dumps(job_data), background=True)
elif current_job.task == "finalizeCurrentCruise":
gm_client = python3_gearman.GearmanClient([self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook('finalizeCurrentCruise'):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task, json.dumps(job_data), background=True)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(self.task['taskID'], results_obj['parts'][-1]['reason'])
else:
self.ovdm.send_msg(self.task['longName'] + ' failed', results_obj['parts'][-1]['reason'])
else:
self.ovdm.set_idle_task(self.task['taskID'])
else:
self.ovdm.set_idle_task(self.task['taskID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s (%s) completed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.cruise_id = self.ovdm.get_cruise_id()
self.cruise_data_transfer = {}
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
self.stop = False
payload_obj = json.loads(current_job.data)
if 'cruiseDataTransferID' in payload_obj['cruiseDataTransfer']:
self.cruise_data_transfer = self.ovdm.get_cruise_data_transfer(payload_obj['cruiseDataTransfer']['cruiseDataTransferID'])
if not self.cruise_data_transfer:
logging.error("could not find configuration data")
return self.on_job_complete(current_job, json.dumps({'parts':[{"partName": "Located Cruise Data Tranfer Data", "result": "Fail", "reason": "Could not find configuration data for cruise data transfer"},{"partName": "Final Verdict", "result": "Fail", "reason": "Could not find configuration data for cruise data transfer"}]}))
self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])
else:
self.cruise_data_transfer = payload_obj['cruiseDataTransfer']
self.cruise_id = self.ovdm.get_cruise_id()
logging.info("Job: %s, %s transfer test started at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))
self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.error("Job: %s, %s transfer test failed at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))
self.send_job_data(current_job, json.dumps([{"partName": "Worker crashed", "result": "Fail", "reason": "Unknown"}]))
if 'cruiseDataTransferID' in self.cruise_data_transfer:
self.ovdm.set_error_cruise_data_transfer_test(self.cruise_data_transfer['cruiseDataTransferID'], 'Worker crashed')
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
if 'cruiseDataTransferID' in self.cruise_data_transfer:
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
self.ovdm.set_error_cruise_data_transfer_test(self.cruise_data_transfer['cruiseDataTransferID'], results_obj['parts'][-1]['reason'])
else:
self.ovdm.clear_error_cruise_data_transfer(self.cruise_data_transfer['cruiseDataTransferID'], self.cruise_data_transfer['status'])
else:
self.ovdm.clear_error_cruise_data_transfer(self.cruise_data_transfer['cruiseDataTransferID'], self.cruise_data_transfer['status'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, %s transfer test completed at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
import argparse
import json
import logging
import sys
import time
from os.path import dirname, realpath
from python3_gearman import GearmanClient
sys.path.append(dirname(dirname(dirname(realpath(__file__)))))
from server.lib.openvdm import OpenVDM
if __name__ == "__main__":
ovdm = OpenVDM()
parser = argparse.ArgumentParser(description='OpenVDM Data Transfer Scheduler')
parser.add_argument('-i', '--interval', default=ovdm.get_transfer_interval(), metavar='interval', type=int, help='interval in minutes')
parser.add_argument('-v', '--verbosity', dest='verbosity',
default=0, action='count',
help='Increase output verbosity')
parsed_args = parser.parse_args()
############################
# Set up logging before we do any other argument parsing (so that we
# can log problems with argument parsing).
LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.job_pid = ''
self.job_info = {}
super().__init__(host_list=[self.ovdm.get_gearman_server()])
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
"""
Class for the current Gearman worker
"""
def __init__(self):
self.stop = False
self.ovdm = OpenVDM()
self.job_pid = ''
self.job_info = {}
super().__init__(host_list=[self.ovdm.get_gearman_server()])
def _get_job_info(self):
"""
Fetch job metadata
"""
collection_system_transfers = self.ovdm.get_collection_system_transfers(
)
for collection_system_transfer in collection_system_transfers:
if collection_system_transfer['pid'] == self.job_pid:
return {
'type': 'collectionSystemTransfer',
'id':
collection_system_transfer['collectionSystemTransferID'],
'name': collection_system_transfer['name'],
'pid': collection_system_transfer['pid']
}
cruise_data_transfers = self.ovdm.get_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
if cruise_data_transfer['pid'] != "0":
return {
'type': 'cruiseDataTransfer',
'id': cruise_data_transfer['cruiseDataTransferID'],
'name': cruise_data_transfer['name'],
'pid': cruise_data_transfer['pid']
}
cruise_data_transfers = self.ovdm.get_required_cruise_data_transfers()
for cruise_data_transfer in cruise_data_transfers:
if cruise_data_transfer['pid'] != "0":
return {
'type': 'cruiseDataTransfer',
'id': cruise_data_transfer['cruiseDataTransferID'],
'name': cruise_data_transfer['name'],
'pid': cruise_data_transfer['pid']
}
tasks = self.ovdm.get_tasks()
for task in tasks:
if task['pid'] != "0":
return {
'type': 'task',
'id': task['taskID'],
'name': task['name'],
'pid': task['pid']
}
return {'type': 'unknown'}
def on_job_execute(self, current_job):
"""
Function run whenever a new job arrives
"""
logging.debug("current_job: %s", current_job)
payload_obj = json.loads(current_job.data)
self.job_pid = payload_obj['pid']
self.job_info = self._get_job_info()
logging.info("Job: %s, Killing PID: %s failed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
return super().on_job_execute(current_job)
def on_job_exception(self, current_job, exc_info):
"""
Function run whenever the current job has an exception
"""
logging.info("Job: %s Killing PID %s failed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
self.send_job_data(
current_job,
json.dumps([{
"partName": "Worker Crashed",
"result": "Fail",
"reason": "Unknown"
}]))
exc_type, _, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.error(exc_type, fname, exc_tb.tb_lineno)
return super().on_job_exception(current_job, exc_info)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s, Killing PID %s completed at: %s",
current_job.handle, self.job_pid,
time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def stop_task(self):
"""
Function to stop the current job
"""
self.stop = True
logging.warning("Stopping current task...")
def quit_worker(self):
"""
Function to quit the worker
"""
self.stop = True
logging.warning("Quitting worker...")
self.shutdown()
help='Increase output verbosity')
parsed_args = parser.parse_args()
############################
# Set up logging before we do any other argument parsing (so that we
# can log problems with argument parsing).
LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity])
ovdm = OpenVDM()
while True:
start_t = datetime.datetime.utcnow()
warehouse_config = ovdm.get_shipboard_data_warehouse_config()
cruise_dir = join(warehouse_config['shipboardDataWarehouseBaseDir'],
ovdm.get_cruise_id())
lowering_id = ovdm.get_lowering_id(
) if ovdm.get_show_lowering_components() else None
lowering_dir = join(cruise_dir,
warehouse_config['loweringDataBaseDir'],
lowering_id) if lowering_id else None