def test_get_sps_packages_moves_anything_if_no_source_file( self, mk_open, mk_path_exists, mk_shutil ): mk_path_exists.return_value = False mk_open.return_value.__enter__.return_value.readlines.return_value = [ "rba v53n1" ] get_sps_packages(**self.kwargs) mk_shutil.move.assert_not_called()
def test_get_sps_packages_ignores_del_command_in_scilista(self): scilista_lines = [ "rba v53n1", "rba 2019nahead", "rsp v10n4s1 del", "rsp v10n4s1", "csp v35nspe del" ] source_filenames = [] scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"]) with scilista_file_path.open("w") as scilista_file: for line in scilista_lines: scilista_file.write(line + "\n") if not line.endswith("del"): source_filenames += [ "_".join([f"2020-01-01-00-0{i}-09-090901"] + line.split()) + ".zip" for i in range(1, 4) ] for filename in source_filenames: zip_filename = pathlib.Path(self.xc_dir_name) / filename with zipfile.ZipFile(zip_filename, "w") as zip_file: zip_file.write(self.test_filepath) sps_packages = get_sps_packages(**self.kwargs) self.assertEqual( sps_packages, [ str(pathlib.Path(self.proc_dir_name).joinpath(filename)) for filename in source_filenames ], )
def test_get_sps_packages_creates_sps_packages_list_file(self): scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"] source_filenames = [] scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"]) with scilista_file_path.open("w") as scilista_file: for line in scilista_lines: scilista_file.write(line + "\n") source_filenames += [ "_".join([f"2020-01-01-00-0{i}-09-090901"] + line.split()) + ".zip" for i in range(1, 4) ] for filename in source_filenames: zip_filename = pathlib.Path(self.xc_dir_name) / filename with zipfile.ZipFile(zip_filename, "w") as zip_file: zip_file.write(self.test_filepath) sps_packages = get_sps_packages(**self.kwargs) package_paths_list = pathlib.Path( self.proc_dir_name) / "sps_packages.lst" self.assertTrue(package_paths_list.is_file()) sps_packages_list = package_paths_list.read_text().split("\n") self.assertEqual( sps_packages_list, [ str(pathlib.Path(self.proc_dir_name).joinpath(filename)) for filename in source_filenames ], )
def test_get_sps_packages_moves_from_xc_dir_to_proc_dir(self): scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"] source_filenames = [] scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"]) with scilista_file_path.open("w") as scilista_file: for line in scilista_lines: scilista_file.write(line + "\n") source_filenames += [ "_".join([f"2020-01-01-00-0{i}-09-090901"] + line.split()) + ".zip" for i in range(1, 4) ] for filename in source_filenames: zip_filename = pathlib.Path(self.xc_dir_name) / filename with zipfile.ZipFile(zip_filename, "w") as zip_file: zip_file.write(self.test_filepath) sps_packages = get_sps_packages(**self.kwargs) for filename in source_filenames: with self.subTest(filename=filename): self.assertTrue( pathlib.Path(self.kwargs["proc_dir_name"]).joinpath( filename).exists()) self.assertEqual( sps_packages, [ str(pathlib.Path(self.proc_dir_name).joinpath(filename)) for filename in source_filenames ], )
def get_sps_packages(conf, **kwargs): """Executa ``pre_sync_documents_to_kernel_operations.get_sps_packages`` com a scilista referente à DagRun. Todos os pacotes obtidos serão armazenados em diretório identificado pelo DAG_RUN_ID junto com a scilista. Caso esta DAG seja reexecutada, os mesmos pacotes serão sincronizados anteriormente serão novamente sincronizados. Armazena os pacotes em XCom para a próxima tarefa. """ _xc_sps_packages_dir = Path(Variable.get("XC_SPS_PACKAGES_DIR")) _proc_sps_packages_dir = Path( Variable.get("PROC_SPS_PACKAGES_DIR")) / kwargs["run_id"] if not _proc_sps_packages_dir.is_dir(): _proc_sps_packages_dir.mkdir() _scilista_file_path = get_scilista_file_path( _xc_sps_packages_dir, _proc_sps_packages_dir, Variable.get("GERAPADRAO_ID_FOR_SCILISTA"), ) _sps_packages = pre_sync_documents_to_kernel_operations.get_sps_packages( _scilista_file_path, _xc_sps_packages_dir, _proc_sps_packages_dir, ) if _sps_packages: kwargs["ti"].xcom_push(key="sps_packages", value=_sps_packages) return True else: return False
def test_get_sps_packages_moves_nothing_if_no_source_file(self): scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"]) package = "rba v53n2" scilista_file_path.write_text(package) source_filenames = [ "2020-05-22-10-00-34-480190_rba_v53n1", "2020-05-22-10-00-34-480190_rba_2019nahead", "2020-05-22-10-00-34-480190_rsp_v10n4s1" ] for filename in source_filenames: zip_filename = pathlib.Path(self.xc_dir_name) / filename with zipfile.ZipFile(zip_filename, "w") as zip_file: zip_file.write(self.test_filepath) get_sps_packages(**self.kwargs) self.assertFalse( pathlib.Path(self.kwargs["proc_dir_name"]).joinpath( "_".join(package.split()) + ".zip").exists())
def test_get_sps_packages_moves_from_xc_dir_to_proc_dir( self, mk_open, mk_path_exists, mk_shutil ): mk_path_exists.return_value = True scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"] mk_open.return_value.__enter__.return_value.readlines.return_value = ( scilista_lines ) with tempfile.TemporaryDirectory() as tmpdirname: self.kwargs["proc_dir_name"] = tmpdirname get_sps_packages(**self.kwargs) for scilista_line in scilista_lines: with self.subTest(scilista_line=scilista_line): filename = "/{}_{}.zip".format(*scilista_line.split()) mk_shutil.move.assert_any_call( self.kwargs["xc_dir_name"] + filename, tmpdirname + filename )
def test_get_sps_packages_must_return_packages_list_if_sps_packages_lst_exists( self): package_paths_list = [ str(pathlib.Path(self.proc_dir_name) / f"package_0{seq}.zip") for seq in range(1, 4) ] package_paths_list_file = pathlib.Path( self.proc_dir_name) / "sps_packages.lst" package_paths_list_file.write_text("\n".join(package_paths_list)) result = get_sps_packages(**self.kwargs) self.assertEqual(result, package_paths_list)
def test_get_sps_packages_must_not_move_brag_if_there_is_ag_in_scilista( self): scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"]) package = "ag 2019nahead" scilista_file_path.write_text(package) source_filenames = ( "2020-05-22-10-00-34-480190_brag_2019nahead.zip\n" "2020-05-22-10-00-34-480190_ag_2019nahead.zip").splitlines() for filename in source_filenames: zip_filename = pathlib.Path(self.xc_dir_name) / filename with zipfile.ZipFile(zip_filename, "w") as zip_file: zip_file.write(self.test_filepath) result = get_sps_packages(**self.kwargs) self.assertEqual([ str( pathlib.Path(self.kwargs["proc_dir_name"]) / "2020-05-22-10-00-34-480190_ag_2019nahead.zip") ], result)
def get_sps_packages(conf, **kwargs): Logger.debug("create_all_subdags IN") sps_packages = pre_sync_documents_to_kernel_operations.get_sps_packages( Variable.get("SCILISTA_FILE_PATH"), Variable.get("XC_SPS_PACKAGES_DIR"), Variable.get("PROC_SPS_PACKAGES_DIR"), ) for sps_package in sps_packages: Logger.info("Triggering an external dag with package %s" % sps_package) now = timezone.utcnow() trigger_dag( dag_id="sync_documents_to_kernel", run_id="manual__%s_%s" % (os.path.basename(sps_package), now.isoformat()), execution_date=now, replace_microseconds=False, conf={"sps_package": sps_package}, ) Logger.debug("create_all_subdags OUT")
def test_read_scilista_from_file(self, mk_open): get_sps_packages(**self.kwargs) mk_open.assert_called_once_with("dir/path/scilista.lst")
def test_get_sps_packages_creates_path_dirs(self, mk_open, MockPath): get_sps_packages(**self.kwargs) MockPath.assert_any_call(self.kwargs["xc_dir_name"]) MockPath.assert_any_call(self.kwargs["proc_dir_name"])