Ejemplo n.º 1
0
    def test_MessagingUtils_cancel(self):
        reduction_run = getReductionRun()

        data_location = DataLocation(file_path="/test/data/path", reduction_run=reduction_run)
        data_location.save()
        reduction_run.data_location.add(data_location)
        reduction_run.save()

        send_called = [False]

        parent = self

        class mock_client(object):
            def __init__(self, brokers, user, password, topics=None, consumer_name='QueueProcessor', client_only=True, use_ssl=ACTIVEMQ['SSL'], ssl_version=3):  pass
            def connect(self): pass
            def stop(self): pass

            def send(self, destination, message, persistent='true', priority=4, delay=None):
                send_called[0] = True
                data_dict = json.loads(message)
                parent.assertTrue('cancel' in data_dict, "Expecting 'cancel' in message")
                parent.assertEqual(data_dict['cancel'], True, "Expecting 'cancel' to be true")

        with patch('autoreduce_webapp.queue_processor.Client', mock_client):
            MessagingUtils().send_cancel(reduction_run)

        self.assertTrue(send_called[0], "Expecting send to be called")
Ejemplo n.º 2
0
    def createRetryRun(self, reductionRun, script=None, variables=None, delay=0, username=None):
        """
        Create a run ready for re-running based on the run provided. 
        If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's.
        If a script (as a string) is supplied then use it, otherwise use the previous run's.
        """
        from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils

        # find the previous run version, so we don't create a duplicate
        last_version = -1
        for run in ReductionRun.objects.filter(experiment=reductionRun.experiment, run_number=reductionRun.run_number):
            last_version = max(last_version, run.run_version)

        try:
            # get the script to use:
            script_text = script if script is not None else reductionRun.script

            # create the run object and save it
            new_job = ReductionRun(
                instrument=reductionRun.instrument,
                run_number=reductionRun.run_number,
                run_name="",
                run_version=last_version + 1,
                experiment=reductionRun.experiment,
                started_by=username,
                status=StatusUtils().get_queued(),
                script=script_text,
            )
            new_job.save()

            reductionRun.retry_run = new_job
            reductionRun.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta(
                seconds=delay if delay else 0
            )
            reductionRun.save()

            # copy the previous data locations
            for data_location in reductionRun.data_location.all():
                new_data_location = DataLocation(file_path=data_location.file_path, reduction_run=new_job)
                new_data_location.save()
                new_job.data_location.add(new_data_location)

            if variables is not None:
                # associate the variables with the new run
                for var in variables:
                    var.reduction_run = new_job
                    var.save()
            else:
                # provide variables if they aren't already
                InstrumentVariablesUtils().create_variables_for_run(new_job)

            return new_job

        except:
            new_job.delete()
            raise
Ejemplo n.º 3
0
    def data_ready(self):
        # Import within method to prevent cylindrical imports
        from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils

        logger.info("Data ready for processing run %s on %s" % (str(self._data_dict['run_number']), self._data_dict['instrument']))
        
        instrument = InstrumentUtils().get_instrument(self._data_dict['instrument'])
        # Activate the instrument if it is currently set to inactive
        if not instrument.is_active:
            instrument.is_active = True
            instrument.save()
            
        status = StatusUtils().get_skipped() if instrument.is_paused else StatusUtils().get_queued()

        last_run = ReductionRun.objects.filter(run_number=self._data_dict['run_number']).order_by('-run_version').first()
        highest_version = last_run.run_version if last_run is not None else -1

        experiment, experiment_created = Experiment.objects.get_or_create(reference_number=self._data_dict['rb_number'])
        if experiment_created:
            experiment.save()
            
        script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0]


        run_version = highest_version+1
        reduction_run = ReductionRun( run_number=self._data_dict['run_number']
                                    , run_version=run_version
                                    , experiment=experiment
                                    , instrument=instrument
                                    , status=status
                                    , script=script_text
                                    )
        reduction_run.save()
        self._data_dict['run_version'] = reduction_run.run_version

        data_location = DataLocation(file_path=self._data_dict['data'], reduction_run=reduction_run)
        data_location.save()

        variables = InstrumentVariablesUtils().create_variables_for_run(reduction_run)
        if not variables:
            logger.warning("No instrument variables found on %s for run %s" % (instrument.name, self._data_dict['run_number']))
        
        
        reduction_script, arguments = ReductionRunUtils().get_script_and_arguments(reduction_run)
        self._data_dict['reduction_script'] = reduction_script
        self._data_dict['reduction_arguments'] = arguments

        if instrument.is_paused:
            logger.info("Run %s has been skipped" % self._data_dict['run_number'])
        else:
            self._client.send('/queue/ReductionPending', json.dumps(self._data_dict), priority=self._priority)
            logger.info("Run %s ready for reduction" % self._data_dict['run_number'])
Ejemplo n.º 4
0
    def test_MessagingUtils_successful(self):
        reduction_run = getReductionRun()

        data_location = DataLocation(file_path="/test/data/path", reduction_run=reduction_run)
        data_location.save()
        reduction_run.data_location.add(data_location)
        reduction_run.save()

        send_called = [False]

        parent = self

        class mock_client(object):
            def __init__(self, brokers, user, password, topics=None, consumer_name='QueueProcessor', client_only=True, use_ssl=ACTIVEMQ['SSL'], ssl_version=3):
                pass

            def connect(self):
                pass
                
            def stop(self):
                pass

            def send(self, destination, message, persistent='true', priority=4, delay=None):
                send_called[0] = True
                data_dict = json.loads(message)
                parent.assertEqual(destination, '/queue/ReductionPending', "Expecting the destination to be '/queue/ReductionPending' but was '%s'." % destination)
                parent.assertEqual(data_dict['data'], '/test/data/path', "Expecting the data path to be '/test/data/path' but was '%s'." % data_dict['data'])
                parent.assertEqual(data_dict['run_number'], reduction_run.run_number, "Expecting the run number to be '%s' but was '%s'." % (reduction_run.run_number, data_dict['run_number']))
                parent.assertEqual(data_dict['run_version'], reduction_run.run_version, "Expecting the run version to be '%s' but was '%s'." % (reduction_run.run_version, data_dict['run_version']))
                parent.assertEqual(data_dict['instrument'], reduction_run.instrument.name, "Expecting the run number to be '%s' but was '%s'." % (reduction_run.instrument.name, data_dict['instrument']))
                parent.assertNotEqual(data_dict['reduction_arguments'], None, "Expecting to find some arguments")
                parent.assertNotEqual(data_dict['reduction_arguments'], {}, "Expecting to find some arguments")
                parent.assertTrue('standard_vars' in data_dict['reduction_arguments'], "Expecting to find some standard_vars.")
                parent.assertTrue('advanced_vars' in data_dict['reduction_arguments'], "Expecting to find some advanced_vars.")

        with patch('autoreduce_webapp.queue_processor.Client', mock_client):
            MessagingUtils().send_pending(reduction_run)

        self.assertTrue(send_called[0], "Expecting send to be called")
Ejemplo n.º 5
0
    def createRetryRun(self,
                       reductionRun,
                       overwrite=None,
                       script=None,
                       variables=None,
                       delay=0,
                       username=None,
                       description=''):
        """
        Create a run ready for re-running based on the run provided. 
        If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's.
        If a script (as a string) is supplied then use it, otherwise use the previous run's.
        """
        from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils

        run_last_updated = reductionRun.last_updated

        if username == 'super':
            username = 1

        # find the previous run version, so we don't create a duplicate
        last_version = -1
        for run in ReductionRun.objects.filter(
                experiment=reductionRun.experiment,
                run_number=reductionRun.run_number):
            last_version = max(last_version, run.run_version)

        try:
            # get the script to use:
            script_text = script if script is not None else reductionRun.script

            # create the run object and save it
            new_job = ReductionRun(instrument=reductionRun.instrument,
                                   run_number=reductionRun.run_number,
                                   run_name=description,
                                   run_version=last_version + 1,
                                   experiment=reductionRun.experiment,
                                   started_by=username,
                                   status=StatusUtils().get_queued(),
                                   script=script_text,
                                   overwrite=overwrite)
            new_job.save()

            reductionRun.retry_run = new_job
            reductionRun.retry_when = timezone.now().replace(
                microsecond=0) + datetime.timedelta(
                    seconds=delay if delay else 0)
            reductionRun.save()

            ReductionRun.objects.filter(id=reductionRun.id).update(
                last_updated=run_last_updated)

            # copy the previous data locations
            for data_location in reductionRun.data_location.all():
                new_data_location = DataLocation(
                    file_path=data_location.file_path, reduction_run=new_job)
                new_data_location.save()
                new_job.data_location.add(new_data_location)

            if variables is not None:
                # associate the variables with the new run
                for var in variables:
                    var.reduction_run = new_job
                    var.save()
            else:
                # provide variables if they aren't already
                InstrumentVariablesUtils().create_variables_for_run(new_job)

            return new_job

        except Exception as e:
            import traceback
            logger.error(traceback.format_exc())
            logger.error(e)
            new_job.delete()
            raise
Ejemplo n.º 6
0
    def createRetryRun(user_id, reduction_run, overwrite=None, script=None,
                       variables=None, delay=0, description=''):
        """
        Create a run ready for re-running based on the run provided.
        If variables (RunVariable) are provided, copy them and associate
        them with the new one, otherwise use the previous run's.
        If a script (as a string) is supplied then use it, otherwise use the previous run's.
        """
        from reduction_variables.utils import InstrumentVariablesUtils

        run_last_updated = reduction_run.last_updated

        # find the previous run version, so we don't create a duplicate
        last_version = -1
        # pylint:disable=no-member
        previous_run = ReductionRun.objects.filter(experiment=reduction_run.experiment,
                                                   run_number=reduction_run.run_number) \
            .order_by("-run_version").first()

        last_version = previous_run.run_version

        # get the script to use:
        script_text = script if script is not None else reduction_run.script

        # create the run object and save it
        new_job = ReductionRun(instrument=reduction_run.instrument,
                               run_number=reduction_run.run_number,
                               run_name=description,
                               run_version=last_version + 1,
                               experiment=reduction_run.experiment,
                               started_by=user_id,
                               status=StatusUtils().get_queued(),
                               script=script_text,
                               overwrite=overwrite)

        # Check record is safe to save
        try:
            new_job.full_clean()
        # pylint:disable=catching-non-exception
        except django.core.exceptions as exception:
            LOGGER.error(traceback.format_exc())
            LOGGER.error(exception)
            raise

        # Attempt to save
        try:
            new_job.save()
        except ValueError as exception:
            # This usually indicates a F.K. constraint mismatch. Maybe we didn't get a record in?
            LOGGER.error(traceback.format_exc())
            LOGGER.error(exception)
            raise

        reduction_run.retry_run = new_job
        reduction_run.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta(
            seconds=delay if delay else 0)
        reduction_run.save()

        # pylint:disable=no-member
        ReductionRun.objects.filter(id=reduction_run.id).update(last_updated=run_last_updated)

        # copy the previous data locations
        # pylint:disable=no-member
        for data_location in reduction_run.data_location.all():
            new_data_location = DataLocation(file_path=data_location.file_path,
                                             reduction_run=new_job)
            new_data_location.save()
            new_job.data_location.add(new_data_location)

        if variables is not None:
            # associate the variables with the new run
            for var in variables:
                var.reduction_run = new_job
                var.save()
        else:
            # provide variables if they aren't already
            InstrumentVariablesUtils().create_variables_for_run(new_job)

        return new_job