def scheduleHeuristicEvaluation(self, job): # Retrieve the heuristic version try: heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0]) except: alert = Alert() alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0] job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return job.markAsRunning() # If the heuristic is already evaluated, remove the results from the database if heuristic_version.evaluated: configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.all()) heuristic_version.evaluation_results.all().delete() heuristic_version.evaluated = False heuristic_version.rank = None heuristic_version.save() for configuration in configurations: configuration.delete() # If the heuristic is currently evaluated, remove the existing results from # the database and cancel the running experiments elif heuristic_version.evaluation_results.count() > 0: for evaluation_results in heuristic_version.evaluation_results.filter(experiment__status=Experiment.STATUS_RUNNING): self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [evaluation_results.experiment.id])) configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING)) heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING).delete() for configuration in configurations: configuration.delete() job.markAsDone() return # Retrieve all the evaluation configurations evaluation_configurations = Configuration.objects.filter(experiment_type=Configuration.EVALUATION, name__startswith='template/') if evaluation_configurations.count() == 0: alert = Alert() alert.message = 'No evaluation configuration found' job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return # Schedule the evaluation experiments for evaluation_configuration in evaluation_configurations: # Create the configuration configuration = Configuration() configuration.name = evaluation_configuration.name.replace('template/', '%s/' % heuristic_version.fullname()) configuration.heuristics = evaluation_configuration.heuristics configuration.experiment_type = evaluation_configuration.experiment_type configuration.task = evaluation_configuration.task configuration.save() # Add the list of heuristic versions for version in evaluation_configuration.heuristics_set.all(): configuration.heuristics_set.add(version) if heuristic_version not in evaluation_configuration.heuristics_set.all(): configuration.heuristics_set.add(heuristic_version) else: configuration.addSetting('PREDICTOR_SETUP/ADDITIONAL_HEURISTICS', heuristic_version.absolutename()) # Add the list of instruments for instrument in evaluation_configuration.instruments_set.all(): configuration.instruments_set.add(instrument) # Save the configuration configuration.save() # Add the settings for setting in evaluation_configuration.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() # Create the evaluation results entry evaluation_results = HeuristicEvaluationResults() evaluation_results.heuristic_version = heuristic_version evaluation_results.evaluation_config = evaluation_configuration evaluation_results.experiment = experiment evaluation_results.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()
def scheduleHeuristicSignatureRecording(self, job): # Retrieve the heuristic version try: heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0]) except: alert = Alert() alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0] job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return job.markAsRunning() # If the heuristic already has a signature, delete it try: signature = heuristic_version.signature[0] except: signature = None if signature is not None: if signature.experiment.isRunning() or signature.experiment.isScheduled: self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [signature.experiment.id])) job.markAsDone() return signature.experiment.configuration.delete() signature.delete() # Retrieve the recording configuration try: configuration_template = Configuration.objects.get(experiment_type=Configuration.SIGNATURE, name__startswith='template/') except: alert = Alert() alert.message = 'No signature recording configuration found' job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return # Create the configuration configuration = Configuration() configuration.name = configuration_template.name.replace('template/', '%s/' % heuristic_version.fullname()) configuration.heuristics = configuration_template.heuristics configuration.experiment_type = configuration_template.experiment_type configuration.task = configuration_template.task configuration.save() # Add the heuristic version configuration.heuristics_set.add(heuristic_version) # Save the configuration configuration.save() # Add the settings for setting in configuration_template.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() # Create the signature signature = HeuristicSignature() signature.heuristic_version = heuristic_version signature.experiment = experiment signature.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()
def schedulePublicExperiments(self, job): job.markAsRunning() # Retrieve the configuration templates for template in Configuration.objects.filter(experiment_type=Configuration.PUBLIC, name__startswith='template/'): # Retrieve the list of heuristic versions already used in a public experiment # based on this template already_used_heuristic_versions = [] previous = None try: previous = Configuration.objects.filter(experiment_type=Configuration.PUBLIC, name__startswith=template.name.replace('template/', '')).order_by('-experiment__creation_date')[0] if previous.experiment.status == Experiment.STATUS_RUNNING: continue already_used_heuristic_versions = list(previous.heuristics_set.all()) except: pass # Ensure that some new heuristics exist since the last public experiment # based on this template current_heuristic_versions = map(lambda x: x.latest_public_version, list(Heuristic.objects.filter(latest_public_version__isnull=False, simple=False))) new_heuristic_versions = filter(lambda x: x not in already_used_heuristic_versions, current_heuristic_versions) if len(new_heuristic_versions) == 0: continue # Cancel the previous experiment if needed if (previous is not None) and (previous.experiment.status == Experiment.STATUS_SCHEDULED): self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [previous.experiment.id])) continue # Create the configuration configuration = Configuration() configuration.name = template.name.replace('template/', '') + '/' + datetime.now().strftime('%Y-%m-%d %H:%M:%S') configuration.heuristics = template.heuristics configuration.experiment_type = template.experiment_type configuration.task = template.task configuration.save() # Add the list of heuristic versions for heuristic_version in current_heuristic_versions: configuration.heuristics_set.add(heuristic_version) # Add the list of instruments for instrument in template.instruments_set.all(): configuration.instruments_set.add(instrument) # Save the configuration configuration.save() # Add the settings for setting in template.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()