class TestEmopPayload(TestCase): @pytest.fixture(autouse=True) def setup_settings(self, tmpdir): self.settings = default_settings() self.input_path = tmpdir.mkdir("input") self.output_path = tmpdir.mkdir("output") self.completed_path = self.output_path.mkdir("completed") self.uploaded_path = self.output_path.mkdir("uploaded") self.settings.payload_input_path = str(self.input_path) self.settings.payload_output_path = str(self.output_path) self.settings.payload_completed_path = str(self.completed_path) self.settings.payload_uploaded_path = str(self.uploaded_path) self.payload = EmopPayload(settings=self.settings, proc_id='1') def test_input_exists_false(self): self.assertEqual(self.payload.input_exists(), False) def test_input_exists_true(self): self.input_path.join("1.json").write("text") self.assertEqual(self.payload.input_exists(), True) def test_output_exists_false(self): self.assertEqual(self.payload.output_exists(), False) def test_output_exists_true(self): self.output_path.join("1.json").write("text") self.assertEqual(self.payload.output_exists(), True) def test_completed_output_exists_false(self): self.assertEqual(self.payload.completed_output_exists(), False) def test_completed_output_exists_true(self): self.completed_path.join("1.json").write("text") self.assertEqual(self.payload.completed_output_exists(), True)
def stage_in_proc_ids(self, proc_ids, wait=0): """ Stage in proc_ids This function will find the necessary data from provided proc_ids and initiate a Globus transfer. Args: proc_ids (list): List of proc_ids to stage in wait (bool): Whether the stage in should wait for the transfer to complete Returns: str: Globus Task ID """ stage_in_files = [] stage_in_data = [] src = self.remote_endpoint dest = self.cluster_endpoint label = "emop-stage-in-files" for proc_id in proc_ids: payload = EmopPayload(self.settings, proc_id) if not payload.input_exists(): logger.error("EmopTransfer: Could not find input payload for proc_id %s", proc_id) continue data = payload.load_input() _files = self._get_stage_in_files_from_data(data) stage_in_files = stage_in_files + _files _stage_in_files = list(set(stage_in_files)) stage_in_data = self._get_stage_in_data(_stage_in_files) task_id = self.start(src=src, dest=dest, data=stage_in_data, label=label, wait=wait) return task_id