def test_successful_job_creation(self): '''Tests successfully processing an ingest that triggers job creation.''' # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', 'data_types': ['type1', 'type2'], }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule( trigger_type='INGEST', configuration=configuration) self.job_type_1.trigger_rule = rule_model self.job_type_1.save() # Call method to test IngestTriggerHandler().process_ingested_source_file( self.source_file, now()) # Check results queue_1 = Queue.objects.get(job_type=self.job_type_1.id) job_exe_1 = JobExecution.objects.select_related().get( pk=queue_1.job_exe_id) job_1 = job_exe_1.job self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id)
def test_successful_recipe_creation(self): """Tests successfully processing an ingest that triggers recipe creation.""" # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule( trigger_type='INGEST', configuration=configuration) self.recipe_type_1.trigger_rule = rule_model self.recipe_type_1.save() # Call method to test IngestTriggerHandler().process_ingested_source_file( self.source_file, now()) # Check results...ensure first job is queued queue_1 = Queue.objects.get(job_type=self.job_type_2.id) job_1 = Job.objects.get(id=queue_1.job_id) self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id) self.assertEqual(job_1.data['output_data'][0]['name'], self.output_name) self.assertEqual(job_1.data['output_data'][0]['workspace_id'], self.workspace.id)
def ready(self): """ Override this method in subclasses to run code when Django starts. """ # Register ingest job type timeout error from job.execution.tasks.job_task import JOB_TYPE_TIMEOUT_ERRORS JOB_TYPE_TIMEOUT_ERRORS['scale-ingest'] = 'ingest-timeout' from ingest.triggers.ingest_trigger_handler import IngestTriggerHandler from trigger.handler import register_trigger_rule_handler # Register ingest trigger rule handler register_trigger_rule_handler(IngestTriggerHandler()) # Registers the Strike monitors with the monitor system import ingest.strike.monitors.factory as factory from ingest.strike.monitors.dir_monitor import DirWatcherMonitor from ingest.strike.monitors.s3_monitor import S3Monitor # Register monitor types factory.add_monitor_type(DirWatcherMonitor) factory.add_monitor_type(S3Monitor) # Registers the scanners with the Scan system import ingest.scan.scanners.factory as factory from ingest.scan.scanners.dir_scanner import DirScanner from ingest.scan.scanners.s3_scanner import S3Scanner # Register monitor types factory.add_scanner_type(DirScanner) factory.add_scanner_type(S3Scanner)
def ready(self): """ Override this method in subclasses to run code when Django starts. """ from ingest.triggers.ingest_trigger_handler import IngestTriggerHandler from trigger.handler import register_trigger_rule_handler # Register ingest trigger rule handler register_trigger_rule_handler(IngestTriggerHandler())
def ready(self): """ Override this method in subclasses to run code when Django starts. """ from ingest.cleanup import IngestJobExecutionCleaner from ingest.strike.cleanup import StrikeJobExecutionCleaner from ingest.triggers.ingest_trigger_handler import IngestTriggerHandler from job.execution.cleanup import REGISTERED_CLEANERS from trigger.handler import register_trigger_rule_handler # Register job execution cleaners for ingest and Strike jobs REGISTERED_CLEANERS['scale-ingest'] = IngestJobExecutionCleaner() REGISTERED_CLEANERS['scale-strike'] = StrikeJobExecutionCleaner() # Register ingest trigger rule handler register_trigger_rule_handler(IngestTriggerHandler())
def _complete_ingest(ingest, status): """Completes the given ingest in an atomic transaction :param ingest: The ingest model :type ingest: :class:`ingest.models.Ingest` :param status: The final status of the ingest :type status: string """ # Atomically mark ingest status and run ingest trigger rules with transaction.atomic(): logger.info('Marking ingest for %s as %s', ingest.file_name, status) ingest.status = status if status == 'INGESTED': ingest.ingest_ended = now() ingest.save() if status == 'INGESTED': IngestTriggerHandler().process_ingested_source_file( ingest.source_file, ingest.ingest_ended)
def ready(self): """ Override this method in subclasses to run code when Django starts. """ from ingest.triggers.ingest_trigger_handler import IngestTriggerHandler from trigger.handler import register_trigger_rule_handler # Register ingest trigger rule handler register_trigger_rule_handler(IngestTriggerHandler()) # Registers the Strike monitors with the monitor system import ingest.strike.monitors.factory as factory from ingest.strike.monitors.dir_monitor import DirWatcherMonitor from ingest.strike.monitors.s3_monitor import S3Monitor # Register monitor types factory.add_monitor_type(DirWatcherMonitor) factory.add_monitor_type(S3Monitor)
def _complete_ingest(ingest, status, source_file): '''Completes the given ingest by marking its :param ingest: The ingest model :type ingest: :class:`ingest.models.Ingest` :param status: The final status of the ingest :type status: str :param source_file: The model of the source file that was ingested :type source_file: :class:`source.models.SourceFile` ''' # TODO: future refactor: this will also be responsible for saving the source file model # Atomically mark ingest status and run ingest trigger rules with transaction.atomic(): logger.info('Marking ingest %i as %s', ingest.id, status) ingest.source_file = source_file ingest.status = status if status == 'INGESTED': ingest.ingest_ended = timezone.now() ingest.save() if status == 'INGESTED': IngestTriggerHandler().process_ingested_source_file(ingest.source_file, ingest.ingest_ended)
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method migrates existing data files into scale. """ logger.info(u'Command starting: migratedata') workspace, workspace_path, local_path, data_types = None, None, None, [] if options['workspace'] is not None and options[ 'workspace_path'] is not None: workspace, workspace_path = options['workspace'], options[ 'workspace_path'] tmp = Workspace.objects.filter(name=workspace) if tmp.count() > 0: workspace = tmp.first() else: workspace = Workspace.objects.get(id=int(workspace)) else: logger.error('Must specify workspace and workspace-path.') return False if options['data_type'] is not None: data_types.extend(options['data_type']) mnt_dirs = None if options['local_path'] is not None: local_path = options['local_path'] else: # mount mnt_dirs = "/tmp", tempfile.mkdtemp() workspace.setup_download_dir(*mnt_dirs) local_path = os.path.join(mnt_dirs[1], workspace_path) logger.info("Ingesting files from %s/%s", workspace.name, workspace_path) filenames = self.generate_file_list(local_path, options['include'], options['exclude']) logger.info("Found %d files", len(filenames)) # prepare for ingest ala strike ingest_records = {} for filename in filenames: logger.info("Generating ingest record for %s" % filename) ingest = Ingest() ingest.file_name = os.path.basename(filename) ingest.file_path = os.path.join( workspace_path, os.path.relpath(filename, local_path)) ingest.transfer_started = datetime.utcfromtimestamp( os.path.getatime(filename)) ingest.file_size = ingest.bytes_transferred = os.path.getsize( filename) ingest.transfer_ended = timezone.now() ingest.media_type = get_media_type(filename) ingest.workspace = workspace for data_type in data_types: ingest.add_data_type_tag(data_type) ingest.status = 'TRANSFERRED' if options['no_commit']: s = IngestDetailsSerializer() logger.info(s.to_representation(ingest)) else: ingest.save() ingest_records[filename] = ingest.id logging.info("Ingests records created") # start ingest tasks for all the files if not options['no_commit']: logging.info("Starting ingest tasks") for filename in filenames: ingest = Ingest.objects.get(id=ingest_records[filename]) logging.info("Processing ingest %s" % ingest.file_name) with transaction.atomic(): ingest.ingest_started = timezone.now() sf = ingest.source_file = SourceFile.create() sf.update_uuid(ingest.file_name) for tag in ingest.get_data_type_tags(): sf.add_data_type_tag(tag) sf.media_type = ingest.media_type sf.file_name = ingest.file_name sf.file_size = ingest.file_size sf.file_path = ingest.file_path sf.workspace = workspace sf.is_deleted = False sf.deleted = None sf.save() sf.set_countries() sf.save() ingest.status = 'INGESTED' ingest.ingest_ended = timezone.now() ingest.source_file = sf ingest.save() IngestTriggerHandler().process_ingested_source_file( ingest.source_file, ingest.ingest_ended) logging.info( "Ingests processed, monitor the queue for triggered jobs.") if mnt_dirs is not None: workspace.cleanup_download_dir(*mnt_dirs) logger.info(u'Command completed: migratedata')