Ejemplo n.º 1
0
class OVDMGearmanWorker(python3_gearman.GearmanWorker):  # pylint: disable=too-many-instance-attributes
    """
    Class for the current Gearman worker
    """
    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.task = None
        self.cruise_id = self.ovdm.get_cruise_id()
        self.lowering_id = self.ovdm.get_lowering_id()
        self.lowering_start_date = self.ovdm.get_lowering_start_date()
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        self.lowering_dir = os.path.join(
            self.
            shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
            self.cruise_id,
            self.shipboard_data_warehouse_config['loweringDataBaseDir'],
            self.lowering_id)

        super().__init__(host_list=[self.ovdm.get_gearman_server()])

    @staticmethod
    def _get_custom_task(current_job):
        """
        Fetch task metadata
        """

        task = list(
            filter(lambda task: task['name'] == current_job.task,
                   CUSTOM_TASKS))
        return task[0] if len(task) > 0 else None

    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """

        logging.debug("current_job: %s", current_job)

        payload_obj = json.loads(current_job.data)

        self.task = self._get_custom_task(
            current_job) if self._get_custom_task(
                current_job) is not None else self.ovdm.get_task_by_name(
                    current_job.task)
        logging.debug("task: %s", self.task)

        if int(self.task['taskID']) > 0:
            self.ovdm.set_running_task(self.task['taskID'], os.getpid(),
                                       current_job.handle)
        # else:
        #     self.ovdm.track_gearman_job(taskLookup[current_job.task], os.getpid(), current_job.handle)

        logging.info("Job: %s (%s) started at: %s", self.task['longName'],
                     current_job.handle, time.strftime("%D %T", time.gmtime()))

        self.cruise_id = payload_obj[
            'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
            )
        self.lowering_id = payload_obj[
            'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id(
            )
        self.lowering_start_date = payload_obj[
            'loweringStartDate'] if 'loweringStartDate' in payload_obj else self.ovdm.get_lowering_start_date(
            )
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        self.lowering_dir = os.path.join(
            self.
            shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
            self.cruise_id,
            self.shipboard_data_warehouse_config['loweringDataBaseDir'],
            self.lowering_id)

        return super().on_job_execute(current_job)

    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s (%s) failed at: %s",
                      self.task['longName'], current_job.handle,
                      time.strftime("%D %T", time.gmtime()))

        self.send_job_data(
            current_job,
            json.dumps([{
                "partName": "Worker crashed",
                "result": "Fail",
                "reason": "Unknown"
            }]))
        if int(self.task['taskID']) > 0:
            self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
        else:
            self.ovdm.send_msg(self.task['longName'] + ' failed',
                               'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)

    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        if len(results_obj['parts']) > 0:
            if results_obj['parts'][-1]['result'] == "Fail":  # Final Verdict
                if int(self.task['taskID']) > 0:
                    self.ovdm.set_error_task(
                        self.task['taskID'],
                        results_obj['parts'][-1]['reason'])
                else:
                    self.ovdm.send_msg(self.task['longName'] + ' failed',
                                       results_obj['parts'][-1]['reason'])
            else:
                if int(self.task['taskID']) > 0:
                    self.ovdm.set_idle_task(self.task['taskID'])
        else:
            if int(self.task['taskID']) > 0:
                self.ovdm.set_idle_task(self.task['taskID'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
                     current_job.handle, time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)

    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")

    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
Ejemplo n.º 2
0
class OVDMGearmanWorker(python3_gearman.GearmanWorker):  # pylint: disable=too-many-instance-attributes
    """
    Class for the current Gearman worker
    """
    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.task = None
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        self.cruise_id = self.ovdm.get_cruise_id()
        self.cruise_dir = os.path.join(
            self.
            shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
            self.cruise_id)
        self.lowering_id = self.ovdm.get_cruise_id()
        self.lowering_dir = os.path.join(
            self.cruise_dir,
            self.shipboard_data_warehouse_config['loweringDataBaseDir'],
            self.lowering_id) if self.lowering_id else None
        self.data_dashboard_dir = os.path.join(
            self.cruise_dir,
            self.ovdm.get_required_extra_directory_by_name('Dashboard_Data')
            ['destDir'])
        self.data_dashboard_manifest_file_path = os.path.join(
            self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN)

        self.collection_system_transfer = {}

        super().__init__(host_list=[self.ovdm.get_gearman_server()])

    @staticmethod
    def get_custom_task(current_job):
        """
        Retrieve task metadata
        """
        task = list(
            filter(lambda task: task['name'] == current_job.task, customTasks))
        return task[0] if len(task) > 0 else None

    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """

        logging.debug("current_job: %s", current_job)

        payload_obj = json.loads(current_job.data)

        self.task = self.get_custom_task(current_job) if self.get_custom_task(
            current_job) is not None else self.ovdm.get_task_by_name(
                current_job.task)
        logging.debug("task: %s", self.task)

        if int(self.task['taskID']) > 0:
            self.ovdm.set_running_task(self.task['taskID'], os.getpid(),
                                       current_job.handle)
        else:
            self.ovdm.track_gearman_job(self.task['longName'], os.getpid(),
                                        current_job.handle)

        logging.info("Job: %s (%s) started at: %s", self.task['longName'],
                     current_job.handle, time.strftime("%D %T", time.gmtime()))

        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        self.cruise_id = payload_obj[
            'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
            )
        self.cruise_dir = os.path.join(
            self.
            shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'],
            self.cruise_id)
        self.lowering_id = payload_obj[
            'loweringID'] if 'loweringID' in payload_obj else self.ovdm.get_lowering_id(
            )
        self.lowering_dir = os.path.join(
            self.cruise_dir,
            self.shipboard_data_warehouse_config['loweringDataBaseDir'],
            self.lowering_id) if self.lowering_id else None
        self.collection_system_transfer = self.ovdm.get_collection_system_transfer(
            payload_obj['collectionSystemTransferID']
        ) if 'collectionSystemTransferID' in payload_obj else {
            'name': "Unknown"
        }
        self.data_dashboard_dir = os.path.join(
            self.cruise_dir,
            self.ovdm.get_required_extra_directory_by_name('Dashboard_Data')
            ['destDir'])
        self.data_dashboard_manifest_file_path = os.path.join(
            self.data_dashboard_dir, DEFAULT_DATA_DASHBOARD_MANIFEST_FN)

        return super().on_job_execute(current_job)

    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s (%s) failed at: %s",
                      self.task['longName'], current_job.handle,
                      time.strftime("%D %T", time.gmtime()))

        self.send_job_data(
            current_job,
            json.dumps([{
                "partName": "Worker crashed",
                "result": "Fail",
                "reason": "Unknown, contact Webb :-)"
            }]))
        if int(self.task['taskID']) > 0:
            self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
        else:
            self.ovdm.send_msg(self.task['longName'] + ' failed',
                               'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)

    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        logging.debug("Preparing subsequent Gearman jobs")

        job_data = {
            'cruiseID': self.cruise_id,
            'loweringID': self.lowering_id,
            'files': results_obj['files']
        }

        if current_job.task == 'updateDataDashboard':

            gm_client = python3_gearman.GearmanClient(
                [self.ovdm.get_gearman_server()])

            payload_obj = json.loads(current_job.data)
            job_data['collectionSystemTransferID'] = payload_obj[
                'collectionSystemTransferID']

            for task in self.ovdm.get_tasks_for_hook(current_job.task):
                logging.info("Adding post task: %s", task)
                gm_client.submit_job(task,
                                     json.dumps(job_data),
                                     background=True)

        elif current_job.task == 'rebuildDataDashboard':

            gm_client = python3_gearman.GearmanClient(
                [self.ovdm.get_gearman_server()])

            for task in self.ovdm.get_tasks_for_hook(current_job.task):
                logging.info("Adding post task: %s", task)
                gm_client.submit_job(task,
                                     json.dumps(job_data),
                                     background=True)

        if len(results_obj['parts']) > 0:
            if results_obj['parts'][-1]['result'] == "Fail":  # Final Verdict
                if int(self.task['taskID']) > 0:
                    self.ovdm.set_error_task(
                        self.task['taskID'],
                        results_obj['parts'][-1]['reason'])
                else:
                    self.ovdm.send_msg(self.task['longName'] + ' failed',
                                       results_obj['parts'][-1]['reason'])
            else:
                if int(self.task['taskID']) > 0:
                    self.ovdm.set_idle_task(self.task['taskID'])
        else:
            if int(self.task['taskID']) > 0:
                self.ovdm.set_idle_task(self.task['taskID'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
                     current_job.handle, time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)

    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")

    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
    """
    Class for the current Gearman worker
    """
    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.cruise_id = self.ovdm.get_cruise_id()
        self.system_status = self.ovdm.get_system_status()
        self.cruise_data_transfer = {}
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        super().__init__(host_list=[self.ovdm.get_gearman_server()])

    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """

        logging.debug("current_job: %s", current_job)

        self.stop = False
        payload_obj = json.loads(current_job.data)

        try:
            self.cruise_data_transfer = self.ovdm.get_cruise_data_transfer(
                payload_obj['cruiseDataTransfer']['cruiseDataTransferID'])

            if not self.cruise_data_transfer:
                return self.on_job_complete(
                    current_job,
                    json.dumps({
                        'parts': [{
                            "partName":
                            "Located Cruise Data Tranfer Data",
                            "result":
                            "Fail",
                            "reason":
                            "Could not find configuration data for cruise data transfer"
                        }],
                        'files': {
                            'new': [],
                            'updated': [],
                            'exclude': []
                        }
                    }))

            if self.cruise_data_transfer['status'] == "1":  #not running
                logging.info(
                    "Transfer job for %s skipped because a transfer for that cruise data destination is already in-progress",
                    self.cruise_data_transfer['name'])
                return self.on_job_complete(
                    current_job,
                    json.dumps({
                        'parts': [{
                            "partName": "Transfer In-Progress",
                            "result": "Fail",
                            "reason": "Transfer is already in-progress"
                        }],
                        'files': {
                            'new': [],
                            'updated': [],
                            'exclude': []
                        }
                    }))

        except Exception as err:
            logging.debug(str(err))
            return self.on_job_complete(
                current_job,
                json.dumps({
                    'parts': [{
                        "partName":
                        "Located Cruise Data Tranfer Data",
                        "result":
                        "Fail",
                        "reason":
                        "Could not find retrieve data for cruise data transfer from OpenVDM API"
                    }],
                    'files': {
                        'new': [],
                        'updated': [],
                        'exclude': []
                    }
                }))

        self.system_status = payload_obj[
            'systemStatus'] if 'systemStatus' in payload_obj else self.ovdm.get_system_status(
            )
        self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])

        if self.system_status == "Off" or self.cruise_data_transfer[
                'enable'] == '0':
            logging.info(
                "Transfer job for %s skipped because that cruise data transfer is currently disabled",
                self.cruise_data_transfer['name'])
            return self.on_job_complete(
                current_job,
                json.dumps({
                    'parts': [{
                        "partName": "Transfer Enabled",
                        "result": "Ignore",
                        "reason": "Transfer is disabled"
                    }],
                    'files': {
                        'new': [],
                        'updated': [],
                        'exclude': []
                    }
                }))

        self.cruise_id = self.ovdm.get_cruise_id()

        logging.info("Job: %s, %s transfer started at: %s", current_job.handle,
                     self.cruise_data_transfer['name'],
                     time.strftime("%D %T", time.gmtime()))

        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )

        return super().on_job_execute(current_job)

    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s, %s transfer failed at: %s", current_job.handle,
                      self.cruise_data_transfer['name'],
                      time.strftime("%D %T", time.gmtime()))

        self.send_job_data(
            current_job,
            json.dumps([{
                "partName": "Worker crashed",
                "result": "Fail",
                "reason": "Unknown"
            }]))
        self.ovdm.set_error_cruise_data_transfer(
            self.cruise_data_transfer['cruiseDataTransferID'],
            'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)

    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        if len(results_obj['parts']) > 0:
            if results_obj['parts'][-1]['result'] == "Fail":  # Final Verdict
                self.ovdm.set_error_cruise_data_transfer(
                    self.cruise_data_transfer['cruiseDataTransferID'],
                    results_obj['parts'][-1]['reason'])
            else:
                self.ovdm.set_idle_cruise_data_transfer(
                    self.cruise_data_transfer['cruiseDataTransferID'])
        else:
            self.ovdm.set_idle_cruise_data_transfer(
                self.cruise_data_transfer['cruiseDataTransferID'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s, %s transfer completed at: %s",
                     current_job.handle, self.cruise_data_transfer['name'],
                     time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)

    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")

    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
Ejemplo n.º 4
0
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
    """
    Class for the current Gearman worker
    """

    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.task = None
        self.cruise_id = self.ovdm.get_cruise_id()
        self.cruise_start_date = self.ovdm.get_cruise_start_date()
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
        self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)
        super().__init__(host_list=[self.ovdm.get_gearman_server()])

    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """
        logging.debug("current_job: %s", current_job)

        payload_obj = json.loads(current_job.data)

        self.stop = False
        self.task = self.ovdm.get_task_by_name(current_job.task)
        logging.debug("task: %s", self.task)

        if int(self.task['taskID']) > 0:
            self.ovdm.set_running_task(self.task['taskID'], os.getpid(), current_job.handle)
#        else:
#            self.ovdm.track_gearman_job(task_lookup[current_job.task], os.getpid(), current_job.handle)

        logging.info("Job: %s (%s) started at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))

        self.cruise_id = payload_obj['cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id()
        self.cruise_start_date = payload_obj['cruiseStartDate'] if 'cruiseStartDate' in payload_obj else self.ovdm.get_cruise_start_date()
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
        self.cruise_dir = os.path.join(self.shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], self.cruise_id)

        return super().on_job_execute(current_job)


    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s (%s) failed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))

        self.send_job_data(current_job, json.dumps([{"partName": "Worker crashed", "result": "Fail", "reason": "Worker crashed"}]))
        if int(self.task['taskID']) > 0:
            self.ovdm.set_error_task(self.task['taskID'], "Worker crashed")
        else:
            self.ovdm.send_msg(self.task['longName'] + ' failed', 'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)


    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        job_data = {
            'cruiseID': self.cruise_id,
            'cruiseStartDate': self.cruise_start_date
        }

        if current_job.task == "setupNewCruise":

            gm_client = python3_gearman.GearmanClient([self.ovdm.get_gearman_server()])

            for task in self.ovdm.get_tasks_for_hook('setupNewCruise'):
                logging.info("Adding post task: %s", task)
                gm_client.submit_job(task, json.dumps(job_data), background=True)

        elif current_job.task == "finalizeCurrentCruise":

            gm_client = python3_gearman.GearmanClient([self.ovdm.get_gearman_server()])

            for task in self.ovdm.get_tasks_for_hook('finalizeCurrentCruise'):
                logging.info("Adding post task: %s", task)
                gm_client.submit_job(task, json.dumps(job_data), background=True)

        if len(results_obj['parts']) > 0:
            if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
                if int(self.task['taskID']) > 0:
                    self.ovdm.set_error_task(self.task['taskID'], results_obj['parts'][-1]['reason'])
                else:
                    self.ovdm.send_msg(self.task['longName'] + ' failed', results_obj['parts'][-1]['reason'])
            else:
                self.ovdm.set_idle_task(self.task['taskID'])
        else:
            self.ovdm.set_idle_task(self.task['taskID'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s (%s) completed at: %s", self.task['longName'], current_job.handle, time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)


    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")


    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
Ejemplo n.º 5
0
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
    """
    Class for the current Gearman worker
    """

    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.cruise_id = self.ovdm.get_cruise_id()
        self.cruise_data_transfer = {}
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()
        super().__init__(host_list=[self.ovdm.get_gearman_server()])


    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """

        logging.debug("current_job: %s", current_job)

        self.stop = False
        payload_obj = json.loads(current_job.data)

        if 'cruiseDataTransferID' in payload_obj['cruiseDataTransfer']:
            self.cruise_data_transfer = self.ovdm.get_cruise_data_transfer(payload_obj['cruiseDataTransfer']['cruiseDataTransferID'])

            if not self.cruise_data_transfer:
                logging.error("could not find configuration data")
                return self.on_job_complete(current_job, json.dumps({'parts':[{"partName": "Located Cruise Data Tranfer Data", "result": "Fail", "reason": "Could not find configuration data for cruise data transfer"},{"partName": "Final Verdict", "result": "Fail", "reason": "Could not find configuration data for cruise data transfer"}]}))

            self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])

        else:
            self.cruise_data_transfer = payload_obj['cruiseDataTransfer']

        self.cruise_id = self.ovdm.get_cruise_id()

        logging.info("Job: %s, %s transfer test started at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))

        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config()

        return super().on_job_execute(current_job)


    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s, %s transfer test failed at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))

        self.send_job_data(current_job, json.dumps([{"partName": "Worker crashed", "result": "Fail", "reason": "Unknown"}]))

        if 'cruiseDataTransferID' in self.cruise_data_transfer:
            self.ovdm.set_error_cruise_data_transfer_test(self.cruise_data_transfer['cruiseDataTransferID'], 'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)



    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        if 'cruiseDataTransferID' in self.cruise_data_transfer:
            if len(results_obj['parts']) > 0:
                if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
                    self.ovdm.set_error_cruise_data_transfer_test(self.cruise_data_transfer['cruiseDataTransferID'], results_obj['parts'][-1]['reason'])
                else:
                    self.ovdm.clear_error_cruise_data_transfer(self.cruise_data_transfer['cruiseDataTransferID'], self.cruise_data_transfer['status'])
            else:
                self.ovdm.clear_error_cruise_data_transfer(self.cruise_data_transfer['cruiseDataTransferID'], self.cruise_data_transfer['status'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s, %s transfer test completed at: %s", current_job.handle, self.cruise_data_transfer['name'], time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)


    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")


    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
Ejemplo n.º 6
0
    LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
    logging.basicConfig(format=LOGGING_FORMAT)

    LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
    parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS))
    logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity])

    ovdm = OpenVDM()

    while True:

        start_t = datetime.datetime.utcnow()

        warehouse_config = ovdm.get_shipboard_data_warehouse_config()
        cruise_dir = join(warehouse_config['shipboardDataWarehouseBaseDir'],
                          ovdm.get_cruise_id())

        lowering_id = ovdm.get_lowering_id(
        ) if ovdm.get_show_lowering_components() else None
        lowering_dir = join(cruise_dir,
                            warehouse_config['loweringDataBaseDir'],
                            lowering_id) if lowering_id else None

        logging.debug("Cruise Directory: %s", cruise_dir)
        logging.debug("Lowering Directory: %s", lowering_dir)

        if isdir(cruise_dir):
            logging.debug("Calculating Cruise Size...")
            cruise_size_proc = subprocess.run(['du', '-sb', cruise_dir],
                                              capture_output=True,
                                              text=True,
class OVDMGearmanWorker(python3_gearman.GearmanWorker):
    """
    Class for the current Gearman worker
    """
    def __init__(self):
        self.stop = False
        self.ovdm = OpenVDM()
        self.cruise_id = self.ovdm.get_cruise_id()
        self.system_status = self.ovdm.get_system_status()
        self.transfer_start_date = None
        self.cruise_data_transfer = self._get_cruise_data_transfer()
        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )
        super().__init__(host_list=[self.ovdm.get_gearman_server()])

    def _get_cruise_data_transfer(self):
        """
        Fetch the ship-to-shore transfer configuration
        """

        try:
            return list(
                filter(lambda transfer: transfer['name'] == 'SSDW',
                       self.ovdm.get_required_cruise_data_transfers()))[0]
        except Exception as err:
            logging.error("Could not find SSDW transfer configuration")
            logging.debug(str(err))
        return None

    def on_job_execute(self, current_job):
        """
        Function run whenever a new job arrives
        """

        logging.debug("current_job: %s", current_job)

        self.stop = False
        payload_obj = json.loads(current_job.data)

        self.cruise_data_transfer = self._get_cruise_data_transfer()
        if not self.cruise_data_transfer:
            return self.on_job_complete(
                current_job,
                json.dumps({
                    'parts': [{
                        "partName": "Transfer Found",
                        "result": "Fail",
                        "reason": "Transfer configuration not found"
                    }],
                    'files': {
                        'new': [],
                        'updated': [],
                        'exclude': []
                    }
                }))

        if 'cruiseDataTransfer' in payload_obj:
            self.cruise_data_transfer.update(payload_obj['cruiseDataTransfer'])

        self.system_status = payload_obj[
            'systemStatus'] if 'systemStatus' in payload_obj else self.ovdm.get_system_status(
            )

        if self.system_status == "Off" or self.cruise_data_transfer[
                'enable'] == '0':
            logging.info(
                "Ship-to-shore Transfer job skipped because ship-to-shore transfers are currently disabled"
            )
            return self.on_job_complete(
                current_job,
                json.dumps({
                    'parts': [{
                        "partName": "Transfer Enabled",
                        "result": "Ignore",
                        "reason": "Transfer is disabled"
                    }],
                    'files': {
                        'new': [],
                        'updated': [],
                        'exclude': []
                    }
                }))

        bw_limit_status = payload_obj[
            'bandwidthLimitStatus'] if 'bandwidthLimitStatus' in payload_obj else self.ovdm.get_ship_to_shore_bw_limit_status(
            )
        if not bw_limit_status:
            self.cruise_data_transfer['bandwidthLimit'] = '0'

        self.cruise_id = payload_obj[
            'cruiseID'] if 'cruiseID' in payload_obj else self.ovdm.get_cruise_id(
            )
        self.transfer_start_date = time.strftime("%Y%m%dT%H%M%SZ",
                                                 time.gmtime())

        logging.info("Job: %s, Ship-to-Shore transfer started at: %s",
                     current_job.handle, time.strftime("%D %T", time.gmtime()))

        self.shipboard_data_warehouse_config = self.ovdm.get_shipboard_data_warehouse_config(
        )

        return super().on_job_execute(current_job)

    def on_job_exception(self, current_job, exc_info):
        """
        Function run whenever the current job has an exception
        """

        logging.error("Job: %s, %s transfer failed at: %s", current_job.handle,
                      self.cruise_data_transfer['name'],
                      time.strftime("%D %T", time.gmtime()))

        self.send_job_data(
            current_job,
            json.dumps([{
                "partName": "Worker crashed",
                "result": "Fail",
                "reason": "Unknown"
            }]))
        self.ovdm.set_error_cruise_data_transfer(
            self.cruise_data_transfer['cruiseDataTransferID'],
            'Worker crashed')

        exc_type, _, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        logging.error(exc_type, fname, exc_tb.tb_lineno)
        return super().on_job_exception(current_job, exc_info)

    def on_job_complete(self, current_job, job_result):
        """
        Function run whenever the current job completes
        """

        results_obj = json.loads(job_result)

        if len(results_obj['parts']) > 0:
            if results_obj['parts'][-1]['result'] == "Fail":  # Final Verdict
                self.ovdm.set_error_cruise_data_transfer(
                    self.cruise_data_transfer['cruiseDataTransferID'],
                    results_obj['parts'][-1]['reason'])
            elif results_obj['parts'][-1]['result'] == "Pass":
                self.ovdm.set_idle_cruise_data_transfer(
                    self.cruise_data_transfer['cruiseDataTransferID'])
        else:
            self.ovdm.set_idle_cruise_data_transfer(
                self.cruise_data_transfer['cruiseDataTransferID'])

        logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
        logging.info("Job: %s, %s transfer completed at: %s",
                     current_job.handle, self.cruise_data_transfer['name'],
                     time.strftime("%D %T", time.gmtime()))

        return super().send_job_complete(current_job, job_result)

    def stop_task(self):
        """
        Function to stop the current job
        """
        self.stop = True
        logging.warning("Stopping current task...")

    def quit_worker(self):
        """
        Function to quit the worker
        """
        self.stop = True
        logging.warning("Quitting worker...")
        self.shutdown()
Ejemplo n.º 8
0
    LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s'
    logging.basicConfig(format=LOGGING_FORMAT)

    LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
    parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS))
    logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity])

    ovdm = OpenVDM()

    while True:

        start_t = datetime.datetime.utcnow()

        warehouse_config = ovdm.get_shipboard_data_warehouse_config()
        cruise_dir = join(warehouse_config['shipboardDataWarehouseBaseDir'], ovdm.get_cruise_id())

        lowering_id = ovdm.get_lowering_id() if ovdm.get_show_lowering_components() else None
        lowering_dir = join(cruise_dir, warehouse_config['loweringDataBaseDir'], lowering_id) if lowering_id else None

        logging.debug("Cruise Directory: %s", cruise_dir)
        logging.debug("Lowering Directory: %s", lowering_dir)

        if isdir(cruise_dir):
            logging.debug("Calculating Cruise Size...")
            cruise_size_proc = subprocess.run(['du','-sb', cruise_dir], capture_output=True, text=True, check=False)
            if cruise_size_proc.returncode == 0:
                logging.info("Cruise Size: %s", cruise_size_proc.stdout.split()[0])
                ovdm.set_cruise_size(cruise_size_proc.stdout.split()[0])

            if lowering_dir and isdir(lowering_dir):