def stage_in_proc_ids(self, proc_ids, wait=0):
        """ Stage in proc_ids

        This function will find the necessary data from provided proc_ids and
        initiate a Globus transfer.

        Args:
            proc_ids (list): List of proc_ids to stage in
            wait (bool): Whether the stage in should wait for the transfer to complete

        Returns:
            str: Globus Task ID
        """
        stage_in_files = []
        stage_in_data = []
        src = self.remote_endpoint
        dest = self.cluster_endpoint
        label = "emop-stage-in-files"
        for proc_id in proc_ids:
            payload = EmopPayload(self.settings, proc_id)
            if not payload.input_exists():
                logger.error("EmopTransfer: Could not find input payload for proc_id %s", proc_id)
                continue
            data = payload.load_input()
            _files = self._get_stage_in_files_from_data(data)
            stage_in_files = stage_in_files + _files

        _stage_in_files = list(set(stage_in_files))
        stage_in_data = self._get_stage_in_data(_stage_in_files)

        task_id = self.start(src=src, dest=dest, data=stage_in_data, label=label, wait=wait)
        return task_id
 def test_stage_out_proc_id_3(self):
     payload = EmopPayload(self.transfer.settings, 'output_payload_1')
     payload.completed_output_exists = MagicMock()
     payload.completed_output_exists.return_value = False
     payload.output_exists = MagicMock()
     payload.output_exists.return_value = False
     retval = self.transfer.stage_out_proc_id('output_payload_1')
     self.assertEqual('', retval)
    def upload_proc_id(self, proc_id):
        payload = EmopPayload(self.settings, proc_id)
        if payload.completed_output_exists():
            filename = payload.completed_output_filename
        elif payload.output_exists():
            filename = payload.output_filename
        else:
            logger.error("EmopUpload: Could not find payload file for proc_id %s" % proc_id)
            return False

        upload_status = self.upload_file(filename=filename)
        return upload_status
    def reserve(self, num_pages, r_filter):
        """Reserve pages for a job

        Reserve page(s) for work by sending PUT request to dashboard API.

        Returns:
            str: The reserved work's proc_id.
        """
        reserve_data = {}
        if r_filter and isinstance(r_filter, dict):
            job_queue = r_filter.copy()
        else:
            job_queue = {}
        job_queue["num_pages"] = num_pages
        if self.settings.operate_on == 'works':
            job_queue["works"] = '1'
        reserve_data["job_queue"] = job_queue
        reserve_request = self.emop_api.put_request("/api/job_queues/reserve", reserve_data)
        if not reserve_request:
            return ""
        requested = reserve_request.get('requested')
        reserved = reserve_request.get('reserved')
        proc_id = reserve_request.get('proc_id')
        results = reserve_request.get('results')
        logger.debug("Requested %s pages, and %s were reserved with proc_id: %s" % (requested, reserved, proc_id))
        logger.debug("Payload: %s" % json.dumps(results, sort_keys=True, indent=4))

        if reserved < 1:
            logger.error("No pages reserved")
            return ""

        self.payload = EmopPayload(self.settings, proc_id)
        self.payload.save_input(results)

        return proc_id
 def setup_settings(self, tmpdir):
     self.settings = default_settings()
     self.input_path = tmpdir.mkdir("input")
     self.output_path = tmpdir.mkdir("output")
     self.completed_path = self.output_path.mkdir("completed")
     self.uploaded_path = self.output_path.mkdir("uploaded")
     self.settings.payload_input_path = str(self.input_path)
     self.settings.payload_output_path = str(self.output_path)
     self.settings.payload_completed_path = str(self.completed_path)
     self.settings.payload_uploaded_path = str(self.uploaded_path)
     self.payload = EmopPayload(settings=self.settings, proc_id='1')
class TestEmopPayload(TestCase):
    @pytest.fixture(autouse=True)
    def setup_settings(self, tmpdir):
        self.settings = default_settings()
        self.input_path = tmpdir.mkdir("input")
        self.output_path = tmpdir.mkdir("output")
        self.completed_path = self.output_path.mkdir("completed")
        self.uploaded_path = self.output_path.mkdir("uploaded")
        self.settings.payload_input_path = str(self.input_path)
        self.settings.payload_output_path = str(self.output_path)
        self.settings.payload_completed_path = str(self.completed_path)
        self.settings.payload_uploaded_path = str(self.uploaded_path)
        self.payload = EmopPayload(settings=self.settings, proc_id='1')

    def test_input_exists_false(self):
        self.assertEqual(self.payload.input_exists(), False)

    def test_input_exists_true(self):
        self.input_path.join("1.json").write("text")
        self.assertEqual(self.payload.input_exists(), True)

    def test_output_exists_false(self):
        self.assertEqual(self.payload.output_exists(), False)

    def test_output_exists_true(self):
        self.output_path.join("1.json").write("text")
        self.assertEqual(self.payload.output_exists(), True)

    def test_completed_output_exists_false(self):
        self.assertEqual(self.payload.completed_output_exists(), False)

    def test_completed_output_exists_true(self):
        self.completed_path.join("1.json").write("text")
        self.assertEqual(self.payload.completed_output_exists(), True)
    def upload_file(self, filename):
        filename_path = os.path.abspath(filename)
        file_basename = os.path.basename(filename_path)
        proc_id, file_ext = os.path.splitext(file_basename)
        payload = EmopPayload(self.settings, proc_id)
        if not os.path.isfile(filename_path):
            logger.error("EmopUpload: Could not find file %s" % filename_path)
            return None

        with open(filename_path) as datafile:
            try:
                data = json.load(datafile)
            except ValueError:
                logger.error("EmopUpload: Invalid JSON file %s" % filename_path)
                return False

        uploaded = self.upload(data)
        if uploaded:
            logger.info("Successfully uploaded payload file %s" % filename_path)
            payload.save_uploaded_output(data)
            return True
        else:
            return False
    def stage_out_proc_id(self, proc_id, wait=0):
        """ Stage out proc_id

        This function will find the necessary data from the provided proc_id and
        initiate a Globus transfer.

        Args:
            proc_id (str): proc_id to stage out
            wait (bool): Whether the stage out should wait for the transfer to complete

        Returns:
            str: Globus Task ID
        """
        payload = EmopPayload(self.settings, proc_id)
        if payload.completed_output_exists():
            filename = payload.completed_output_filename
        elif payload.output_exists():
            filename = payload.output_filename
        elif payload.uploaded_output_exists():
            filename = payload.uploaded_output_filename
        else:
            logger.error("EmopTransfer: Could not find payload file for proc_id %s" % proc_id)
            return ''

        data = payload.load(filename)
        if not data:
            logger.error("EmopTransfer: Unable to load payload data")
            return ''
        stage_out_data = self._get_stage_out_data(data)
        src = self.cluster_endpoint
        dest = self.remote_endpoint
        label = "emop-stage-out-%s" % proc_id

        logger.debug("Stage out files:\n%s", json.dumps(stage_out_data, indent=4, sort_keys=True))

        task_id = self.start(src=src, dest=dest, data=stage_out_data, label=label, wait=wait)
        return task_id
    def __init__(self, config_path, proc_id):
        """ Initialize EmopRun object and attributes

        Args:
            config_path (str): path to application config file
            proc_id (str or int): proc-id of this run
        """
        super(self.__class__, self).__init__(config_path)
        self.proc_id = proc_id
        self.payload = EmopPayload(self.settings, proc_id)
        self.scheduler = EmopScheduler.get_scheduler_instance(name=self.settings.scheduler, settings=self.settings)
        self.results = {}
        self.jobs_completed = []
        self.jobs_failed = []
        self.page_results = []
        self.postproc_results = []
        self.font_training_results = []
        self.extra_transfers = []
class EmopSubmit(EmopBase):

    def __init__(self, config_path):
        """ Initialize EmopSubmit object and attributes

        Args:
            config_path (str): path to application config file
        """
        super(self.__class__, self).__init__(config_path)
        self.scheduler = EmopScheduler.get_scheduler_instance(name=self.settings.scheduler, settings=self.settings)

    def optimize_submit(self, page_count, running_job_count, sim=False):
        """Determine optimal job submission

        This function attempts to determine the best number of jobs
        and how many pages per job should be submitted to the scheduler.

        This function does not return a value but sets the num_jobs and
        pages_per_job attributes.

        Args:
            page_count (int): Number of pages needing to be processed
            running_job_count (int): Number of active jobs

        Returns:
            list: First value is number of jobs and second value
                is number of pages per job.
        """
        num_jobs = 0
        pages_per_job = 1
        job_slots_available = int(self.settings.max_jobs - running_job_count)
        run_option_a = float(page_count) / float(job_slots_available)
        run_option_b = float(self.settings.max_job_runtime) / float(self.settings.avg_page_runtime)
        run_option_c = float(self.settings.min_job_runtime) / float(self.settings.avg_page_runtime)
        logger.debug("JobSlotsAvailable: %s, PageCount: %s" % (job_slots_available, page_count))
        logger.debug("RunOptA: %s , RunOptB: %s, RunOptC: %s" % (run_option_a, run_option_b, run_option_c))

        # max pages per job > pages in max time
        if run_option_a > run_option_b:
            num_jobs = job_slots_available
            pages_per_job = run_option_b
        # Pages less than pages in min time
        elif page_count < run_option_c:
            num_jobs = page_count / run_option_c
            pages_per_job = page_count
        # max pages per job < pages in min time
        elif run_option_a < run_option_c:
            num_jobs = page_count / run_option_c
            pages_per_job = run_option_c
        # max pages per job
        else:
            # TODO: In some cases num_jobs will exceed max_jobs value
            num_jobs = page_count / run_option_a
            pages_per_job = run_option_a

        # Convert values to integers
        num_jobs = int(num_jobs)
        pages_per_job = int(pages_per_job)

        # Incase num_jobs and pages_per_job were type casted to 0
        if not num_jobs:
            num_jobs = 1
        if not pages_per_job:
            pages_per_job = 1

        # Case where num_jobs > page_count
        if num_jobs > page_count or (num_jobs*pages_per_job) > page_count:
            if page_count > job_slots_available:
                num_jobs = job_slots_available
                pages_per_job = int(page_count/num_jobs)
            else:
                num_jobs = page_count
                pages_per_job = 1

        expected_runtime = pages_per_job * self.settings.avg_page_runtime
        expected_runtime_msg = "Expected job runtime: %s seconds" % expected_runtime
        if sim:
            logger.info(expected_runtime_msg)
        else:
            logger.debug(expected_runtime_msg)

        # total_pages_to_run = num_jobs * pages_per_job

        optimal_submit_msg = "Optimal submission is %s jobs with %s pages per job" % (num_jobs, pages_per_job)
        if sim:
            logger.info(optimal_submit_msg)
        else:
            logger.debug(optimal_submit_msg)

        return num_jobs, pages_per_job

    def reserve(self, num_pages, r_filter):
        """Reserve pages for a job

        Reserve page(s) for work by sending PUT request to dashboard API.

        Returns:
            str: The reserved work's proc_id.
        """
        reserve_data = {}
        if r_filter and isinstance(r_filter, dict):
            job_queue = r_filter.copy()
        else:
            job_queue = {}
        job_queue["num_pages"] = num_pages
        if self.settings.operate_on == 'works':
            job_queue["works"] = '1'
        reserve_data["job_queue"] = job_queue
        reserve_request = self.emop_api.put_request("/api/job_queues/reserve", reserve_data)
        if not reserve_request:
            return ""
        requested = reserve_request.get('requested')
        reserved = reserve_request.get('reserved')
        proc_id = reserve_request.get('proc_id')
        results = reserve_request.get('results')
        logger.debug("Requested %s pages, and %s were reserved with proc_id: %s" % (requested, reserved, proc_id))
        logger.debug("Payload: %s" % json.dumps(results, sort_keys=True, indent=4))

        if reserved < 1:
            logger.error("No pages reserved")
            return ""

        self.payload = EmopPayload(self.settings, proc_id)
        self.payload.save_input(results)

        return proc_id

    def set_job_id(self, proc_id, job_id):
        """Sends JobID back to dashboard
        """
        data = {
            "job_queue": {
                "proc_id": proc_id,
                "job_id": job_id,
            }
        }
        set_job_id_request = self.emop_api.put_request('/api/job_queues/set_job_id', data)
        return True
class EmopRun(EmopBase):

    def __init__(self, config_path, proc_id):
        """ Initialize EmopRun object and attributes

        Args:
            config_path (str): path to application config file
            proc_id (str or int): proc-id of this run
        """
        super(self.__class__, self).__init__(config_path)
        self.proc_id = proc_id
        self.payload = EmopPayload(self.settings, proc_id)
        self.scheduler = EmopScheduler.get_scheduler_instance(name=self.settings.scheduler, settings=self.settings)
        self.results = {}
        self.jobs_completed = []
        self.jobs_failed = []
        self.page_results = []
        self.postproc_results = []
        self.font_training_results = []
        self.extra_transfers = []

    def append_result(self, job, results, failed=False):
        """Append a page's results to job's results payload

        The results are saved to the output JSON file so that the status
        of each page is saved upon failure or success.

        Args:
            job (EmopJob): EmopJob object
            results (str): The error output of a particular process
            failed (bool, optional): Sets if the result is a failure
        """
        if failed:
            results_ext = "%s JOB %s: %s" % (self.scheduler.name, self.scheduler.job_id, results)
            logger.error(results_ext)
            if self.settings.operate_on == 'works':
                for j in job.jobs:
                    self.jobs_failed.append({"id": j.id, "results": results_ext})
            else:
                self.jobs_failed.append({"id": job.id, "results": results_ext})
        else:
            if self.settings.operate_on == 'works':
                for j in job.jobs:
                    self.jobs_completed.append(j.id)
            else:
                self.jobs_completed.append(job.id)

        # TODO: Do we need to handle adding page_results and postproc_results differently??
        _jobs = []
        if self.settings.operate_on == 'works':
            for j in job.jobs:
                _jobs.append(j)
        else:
            _jobs = [job]
        for j in _jobs:
            if j.page_result.has_data():
                self.page_results.append(j.page_result.to_dict())
            if j.postproc_result.has_data():
                self.postproc_results.append(j.postproc_result.to_dict())
            if j.font_training_result.has_data():
                self.font_training_results.append(j.font_training_result.to_dict())
            if j.extra_transfers:
                self.extra_transfers = self.extra_transfers + j.extra_transfers

        current_results = self.get_results()
        self.payload.save_output(data=current_results, overwrite=True)

    def get_results(self):
        """Get this object's results

        Returns:
            dict: Results to be used as payload to API
        """
        job_queues_data = {
            "completed": self.jobs_completed,
            "failed": self.jobs_failed,
        }
        data = {
            "job_queues": job_queues_data,
            "page_results": self.page_results,
            "postproc_results": self.postproc_results,
            "font_training_results": self.font_training_results,
            "extra_transfers": self.extra_transfers
        }

        return data

    @EmopBase.run_timing
    def do_process(self, obj, job, **kwargs):
        """ Run a process

        This function is intended to handle calling and getting the
        success or failure of a job's post process.

        If a process does not return an exitcode of 0 then a failure has occurred
        and the stderr is added to the job's results.

        Args:
            obj (object): The class of a process
            job (EmopJob): EmopJob object
            **kwargs: Arbitrary keyword arguments.

        Returns:
            bool: True if successful, False otherwise.
        """
        klass = obj.__class__.__name__
        if self.settings.controller_skip_existing and not obj.should_run():
            logger.info("Skipping %s job [%s]" % (klass, job.id))
            return True
        result = obj.run(**kwargs)
        if result.exitcode != 0:
            err = "%s Failed: %s" % (klass, result.stderr)
            # TODO need to rework so failed doesn't mean done
            self.append_result(job=job, results=err, failed=True)
            return False
        else:
            return True

    @EmopBase.run_timing
    def do_ocr(self, job):
        """Run the OCR

        The actual OCR class is called from here.  Based on the value
        of the ocr_engine, a different class will be called.

        The ocr_results returned by the OCR class are used to determine if
        the ocr was successful and the results are appended to global results.

        Args:
            job (EmopJob): EmopJob object

        Returns:
            bool: True if successful, False otherwise.
        """
        logger.info(
            "Got job [%s] - Batch: %s JobType: %s OCR Engine: %s" %
            (job.id, job.batch_job.name, job.batch_job.job_type, job.batch_job.ocr_engine)
        )

        # OCR #
        ocr_engine = job.batch_job.ocr_engine
        if ocr_engine == "tesseract":
            ocr = Tesseract(job=job)
        elif ocr_engine == "ocular":
            ocr = OcularTranscribe(job=job)
        else:
            ocr_engine_err = "OCR with %s not yet supported" % ocr_engine
            self.append_result(job=job, results=ocr_engine_err, failed=True)
            return False

        if self.settings.controller_skip_existing and not ocr.should_run():
            logger.info("Skipping OCR job [%s]" % job.id)
            return True
        ocr_result = ocr.run()

        if ocr_result.exitcode != 0:
            ocr_err = "%s OCR Failed: %s" % (ocr_engine, ocr_result.stderr)
            self.append_result(job=job, results=ocr_err, failed=True)
            return False
        else:
            return True

    def do_postprocesses(self, job):
        """Run the post processes

        Each post process class is called from here.

        Currently the steps are executed in the following order:
            * Denoise
            * MultiColumnSkew
            * XML_To_Text
            * PageEvaluator
            * PageCorrector
            * JuxtaCompare (postprocess)
            * JuxtaCompare - COMMENTED OUT
            * RetasCompare (postprocess)
            * RetasCompare - COMMENTED OUT

        If any step fails, the function terminates and returns False.

        Args:
            job (EmopJob): EmopJob object

        Returns:
            bool: True if successful, False otherwise.
        """
        # DeNoise #
        if self.settings.denoise_enabled:
            denoise = Denoise(job=job)
            denoise_proc = self.do_process(obj=denoise, job=job)
            if not denoise_proc:
                return False

        # MultiColumnSkew #
        if self.settings.denoise_enabled and self.settings.multi_column_skew_enabled:
            multi_column_skew = MultiColumnSkew(job=job)
            multi_column_skew_proc = self.do_process(obj=multi_column_skew, job=job)
            if not multi_column_skew_proc:
                return False

        # _IDHMC.xml to _IDHMC.txt #
        if self.settings.denoise_enabled:
            xml_to_text = XML_To_Text(job=job)
            xml_to_text_proc = self.do_process(obj=xml_to_text, job=job)
            if not xml_to_text_proc:
                return False

        # PageEvaluator #
        if self.settings.page_evaluator_enabled:
            page_evaluator = PageEvaluator(job=job)
            page_evaluator_proc = self.do_process(obj=page_evaluator, job=job)
            if not page_evaluator_proc:
                return False

        # PageCorrector #
        if self.settings.page_corrector_enabled:
            page_corrector = PageCorrector(job=job)
            page_corrector_proc = self.do_process(obj=page_corrector, job=job)
            if not page_corrector_proc:
                return False

        # JuxtaCompare postprocess and OCR output #
        juxta_compare = JuxtaCompare(job=job)
        juxta_compare_proc_pp = self.do_process(obj=juxta_compare, job=job, postproc=True)
        if not juxta_compare_proc_pp:
            return False
        # juxta_compare_proc = self.do_process(obj=juxta_compare, job=job, postproc=False)
        # if not juxta_compare_proc:
        #     return False

        # RetasCompare postprocess and OCR output #
        # retas_compare = RetasCompare(job=job)
        # retas_compare_proc_pp = self.do_process(obj=retas_compare, job=job, postproc=True)
        # if not retas_compare_proc_pp:
        #    return False
        # retas_compare_proc = self.do_process(obj=retas_compare, job=job, postproc=False)
        # if not retas_compare_proc:
        #     return False

        return True

    @EmopBase.run_timing
    def do_job(self, job):
        """Execute the parts of a page's job

        Args:
            job (EmopJob): EmopJob object

        Returns:
            bool: True if successful, False otherwise.
        """
        if not self.do_ocr(job=job):
            return False
        if not self.do_postprocesses(job=job):
            return False
        return True

    @EmopBase.run_timing
    def do_training(self, job):
        """Execute Training"""
        logger.info(
            "Got job [%s] - Batch: %s JobType: %s OCR Engine: %s" %
            (job.id, job.batch_job.name, job.batch_job.job_type, job.batch_job.ocr_engine)
        )

        # OCR #
        training_engine = job.batch_job.ocr_engine
        if training_engine == "ocular":
            training = OcularFontTraining(job=job)
        else:
            training_engine_err = "Training with %s not yet supported" % training_engine
            self.append_result(job=job, results=training_engine_err, failed=True)
            return False

        #if self.settings.controller_skip_existing and not ocr.should_run():
        #    logger.info("Skipping OCR job [%s]" % job.id)
        #    return True
        training_result = training.run()

        if training_result.exitcode != 0:
            training_err = "%s Training Failed: %s" % (training_engine, training_result.stderr)
            self.append_result(job=job, results=training_err, failed=True)
            return False
        else:
            return True

    @EmopBase.run_timing
    def run(self, force=False):
        """Run the EmopJob

        This function is intended to be what's called by external scripts
        like emopcmd.py to start all work.

        Based on the payload's data, all pages are iterated over from here.

        Once the loop of all jobs is complete the final results are saved
        to a file as completed payload

        Args:
            force (bool): Run even if output file exists.

        Returns:
            bool: True if successful, False otherwise.
        """
        global instance
        global job_ids
        data = self.payload.load_input()
        logger.debug("Payload: \n%s" % json.dumps(data, sort_keys=True, indent=4))

        if not data:
            logger.error("No payload data to load.")
            return False
        if not force:
            if self.payload.output_exists():
                logger.error("Output file %s already exists." % self.payload.output_filename)
                return False
            if self.payload.completed_output_exists():
                logger.error("Output file %s already exists." % self.payload.completed_output_filename)
                return False

        # Assign global variables and respond to signals
        for job in data:
            job_ids.append(job["id"])
        instance = self
        signal.signal(signal.SIGUSR1, signal_exit)

        # Loop over data to create EmopJob records
        emop_jobs = {}
        for job in data:
            emop_job = EmopJob(job_data=job, settings=self.settings, scheduler=self.scheduler)
            if self.settings.operate_on == 'works':
                work_id = emop_job.work.id
                if work_id in emop_jobs:
                    _emop_job = emop_jobs[work_id]
                    _emop_job.jobs.append(emop_job)
                    continue
                else:
                    emop_job.jobs = [emop_job]
                    emop_jobs[work_id] = emop_job
            else:
                page_id = emop_job.page.id
                emop_jobs[page_id] = emop_job

        # Loop over jobs to perform actual work
        for emop_job_id, emop_job in emop_jobs.iteritems():
            if emop_job.batch_job.job_type == "ocr":
                job_succcessful = self.do_job(job=emop_job)
                if not job_succcessful:
                    continue
                # Append successful completion of page #
                self.append_result(job=emop_job, results=None, failed=False)
            elif emop_job.batch_job.job_type == 'font training':
                job_successful = self.do_training(job=emop_job)
                if not job_successful:
                    continue
                # Append successful completion #
                self.append_result(job=emop_job, results=None, failed=False)
            else:
                logger.error("JobType of %s is not yet supported." % emop_job.batch_job.job_type)
                return False

        logger.debug("Payload: \n%s" % json.dumps(self.get_results(), sort_keys=True, indent=4))
        self.payload.save_completed_output(data=self.get_results(), overwrite=force)
        return True