def test_is_task_queued_has_tasks(self): # Arrange file_path = to_normalized_unprocessed_file_path( 'bucket/file_path.csv', GcsfsDirectIngestFileType.INGEST_VIEW) gcsfs_args = \ GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=GcsfsFilePath.from_absolute_path(file_path)) full_task_name = \ _build_task_id(_REGION.region_code, gcsfs_args.task_id_tag()) info = ProcessIngestJobCloudTaskQueueInfo( queue_name='queue_name', task_names=[ 'projects/path/to/random_task', f'projects/path/to/{full_task_name}' ]) file_path = to_normalized_unprocessed_file_path( 'bucket/file_path.csv', GcsfsDirectIngestFileType.INGEST_VIEW) gcsfs_args = \ GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=GcsfsFilePath.from_absolute_path(file_path)) # Act gcsfs_args_queued = info.is_task_queued(_REGION, gcsfs_args) # Assert self.assertTrue(gcsfs_args_queued)
def test_info_single_task(self) -> None: # Arrange gcsfs_args = GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=self.ingest_view_file_path, ) full_task_name = _build_task_id(_REGION.region_code, DirectIngestInstance.PRIMARY, gcsfs_args.task_id_tag()) info = ProcessIngestJobCloudTaskQueueInfo( queue_name="queue_name", task_names=[ "projects/path/to/random_task", f"projects/path/to/{full_task_name}", ], ) gcsfs_args = GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=self.ingest_view_file_path, ) # Act gcsfs_args_queued = info.is_task_queued(_REGION, gcsfs_args) # Assert self.assertTrue(gcsfs_args_queued) self.assertTrue( info.tasks_for_instance(_REGION.region_code, DirectIngestInstance.PRIMARY)) self.assertFalse( info.tasks_for_instance(_REGION.region_code, DirectIngestInstance.SECONDARY))
def get_process_job_queue_info( self, region: Region ) -> ProcessIngestJobCloudTaskQueueInfo: return ProcessIngestJobCloudTaskQueueInfo( queue_name="process", task_names=[t[0] for t in self.process_job_tasks] )
def get_process_job_queue_info( self, region: Region) -> ProcessIngestJobCloudTaskQueueInfo: with self.process_job_queue.all_tasks_mutex: task_names = self.process_job_queue.get_unfinished_task_names_unsafe( ) return ProcessIngestJobCloudTaskQueueInfo( queue_name=self.process_job_queue.name, task_names=task_names)
def test_info_no_tasks(self) -> None: # Arrange info = ProcessIngestJobCloudTaskQueueInfo(queue_name="queue_name", task_names=[]) gcsfs_args = GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=self.ingest_view_file_path, ) # Act gcsfs_args_queued = info.is_task_queued(_REGION, gcsfs_args) # Assert self.assertFalse(gcsfs_args_queued) self.assertFalse(info.is_task_queued(_REGION, gcsfs_args)) for instance in DirectIngestInstance: self.assertFalse( info.tasks_for_instance(_REGION.region_code, instance))
def test_is_task_queued_no_tasks(self): # Arrange info = ProcessIngestJobCloudTaskQueueInfo(queue_name='queue_name', task_names=[]) file_path = to_normalized_unprocessed_file_path( 'bucket/file_path.csv', GcsfsDirectIngestFileType.INGEST_VIEW) args = IngestArgs(ingest_time=datetime.datetime.now()) gcsfs_args = \ GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=GcsfsFilePath.from_absolute_path(file_path)) # Act basic_args_queued = info.is_task_queued(_REGION, args) gcsfs_args_queued = info.is_task_queued(_REGION, gcsfs_args) # Assert self.assertFalse(basic_args_queued) self.assertFalse(gcsfs_args_queued) self.assertFalse(info.is_task_queued(_REGION, gcsfs_args))
def test_info_tasks_both_instances(self) -> None: # Arrange gcsfs_args = GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=self.ingest_view_file_path, ) full_task_names = [ _build_task_id( _REGION.region_code, ingest_instance, gcsfs_args.task_id_tag(), ) for ingest_instance in DirectIngestInstance ] info = ProcessIngestJobCloudTaskQueueInfo( queue_name="queue_name", task_names=[ "projects/path/to/random_task", ] + [ f"projects/path/to/{full_task_name}" for full_task_name in full_task_names ], ) gcsfs_args = GcsfsIngestArgs( ingest_time=datetime.datetime.now(), file_path=self.ingest_view_file_path, ) # Act gcsfs_args_queued = info.is_task_queued(_REGION, gcsfs_args) # Assert self.assertTrue(gcsfs_args_queued) for ingest_instance in DirectIngestInstance: self.assertTrue( info.tasks_for_instance(_REGION.region_code, ingest_instance))