def test_get_sps_packages_moves_anything_if_no_source_file(
self, mk_open, mk_path_exists, mk_shutil
):
mk_path_exists.return_value = False
mk_open.return_value.__enter__.return_value.readlines.return_value = [
"rba v53n1"
]
get_sps_packages(**self.kwargs)
mk_shutil.move.assert_not_called()
def test_get_sps_packages_ignores_del_command_in_scilista(self):
scilista_lines = [
"rba v53n1", "rba 2019nahead", "rsp v10n4s1 del", "rsp v10n4s1",
"csp v35nspe del"
]
source_filenames = []
scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"])
with scilista_file_path.open("w") as scilista_file:
for line in scilista_lines:
scilista_file.write(line + "\n")
if not line.endswith("del"):
source_filenames += [
"_".join([f"2020-01-01-00-0{i}-09-090901"] +
line.split()) + ".zip" for i in range(1, 4)
]
for filename in source_filenames:
zip_filename = pathlib.Path(self.xc_dir_name) / filename
with zipfile.ZipFile(zip_filename, "w") as zip_file:
zip_file.write(self.test_filepath)
sps_packages = get_sps_packages(**self.kwargs)
self.assertEqual(
sps_packages,
[
str(pathlib.Path(self.proc_dir_name).joinpath(filename))
for filename in source_filenames
],
)
def test_get_sps_packages_creates_sps_packages_list_file(self):
scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"]
source_filenames = []
scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"])
with scilista_file_path.open("w") as scilista_file:
for line in scilista_lines:
scilista_file.write(line + "\n")
source_filenames += [
"_".join([f"2020-01-01-00-0{i}-09-090901"] + line.split())
+ ".zip" for i in range(1, 4)
]
for filename in source_filenames:
zip_filename = pathlib.Path(self.xc_dir_name) / filename
with zipfile.ZipFile(zip_filename, "w") as zip_file:
zip_file.write(self.test_filepath)
sps_packages = get_sps_packages(**self.kwargs)
package_paths_list = pathlib.Path(
self.proc_dir_name) / "sps_packages.lst"
self.assertTrue(package_paths_list.is_file())
sps_packages_list = package_paths_list.read_text().split("\n")
self.assertEqual(
sps_packages_list,
[
str(pathlib.Path(self.proc_dir_name).joinpath(filename))
for filename in source_filenames
],
)
def test_get_sps_packages_moves_from_xc_dir_to_proc_dir(self):
scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"]
source_filenames = []
scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"])
with scilista_file_path.open("w") as scilista_file:
for line in scilista_lines:
scilista_file.write(line + "\n")
source_filenames += [
"_".join([f"2020-01-01-00-0{i}-09-090901"] + line.split())
+ ".zip" for i in range(1, 4)
]
for filename in source_filenames:
zip_filename = pathlib.Path(self.xc_dir_name) / filename
with zipfile.ZipFile(zip_filename, "w") as zip_file:
zip_file.write(self.test_filepath)
sps_packages = get_sps_packages(**self.kwargs)
for filename in source_filenames:
with self.subTest(filename=filename):
self.assertTrue(
pathlib.Path(self.kwargs["proc_dir_name"]).joinpath(
filename).exists())
self.assertEqual(
sps_packages,
[
str(pathlib.Path(self.proc_dir_name).joinpath(filename))
for filename in source_filenames
],
)
def get_sps_packages(conf, **kwargs):
"""Executa ``pre_sync_documents_to_kernel_operations.get_sps_packages`` com a
scilista referente à DagRun. Todos os pacotes obtidos serão armazenados em
diretório identificado pelo DAG_RUN_ID junto com a scilista. Caso esta DAG seja
reexecutada, os mesmos pacotes serão sincronizados anteriormente serão novamente
sincronizados.
Armazena os pacotes em XCom para a próxima tarefa.
"""
_xc_sps_packages_dir = Path(Variable.get("XC_SPS_PACKAGES_DIR"))
_proc_sps_packages_dir = Path(
Variable.get("PROC_SPS_PACKAGES_DIR")) / kwargs["run_id"]
if not _proc_sps_packages_dir.is_dir():
_proc_sps_packages_dir.mkdir()
_scilista_file_path = get_scilista_file_path(
_xc_sps_packages_dir,
_proc_sps_packages_dir,
Variable.get("GERAPADRAO_ID_FOR_SCILISTA"),
)
_sps_packages = pre_sync_documents_to_kernel_operations.get_sps_packages(
_scilista_file_path,
_xc_sps_packages_dir,
_proc_sps_packages_dir,
)
if _sps_packages:
kwargs["ti"].xcom_push(key="sps_packages", value=_sps_packages)
return True
else:
return False
def test_get_sps_packages_moves_nothing_if_no_source_file(self):
scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"])
package = "rba v53n2"
scilista_file_path.write_text(package)
source_filenames = [
"2020-05-22-10-00-34-480190_rba_v53n1",
"2020-05-22-10-00-34-480190_rba_2019nahead",
"2020-05-22-10-00-34-480190_rsp_v10n4s1"
]
for filename in source_filenames:
zip_filename = pathlib.Path(self.xc_dir_name) / filename
with zipfile.ZipFile(zip_filename, "w") as zip_file:
zip_file.write(self.test_filepath)
get_sps_packages(**self.kwargs)
self.assertFalse(
pathlib.Path(self.kwargs["proc_dir_name"]).joinpath(
"_".join(package.split()) + ".zip").exists())
def test_get_sps_packages_moves_from_xc_dir_to_proc_dir(
self, mk_open, mk_path_exists, mk_shutil
):
mk_path_exists.return_value = True
scilista_lines = ["rba v53n1", "rba 2019nahead", "rsp v10n4s1"]
mk_open.return_value.__enter__.return_value.readlines.return_value = (
scilista_lines
)
with tempfile.TemporaryDirectory() as tmpdirname:
self.kwargs["proc_dir_name"] = tmpdirname
get_sps_packages(**self.kwargs)
for scilista_line in scilista_lines:
with self.subTest(scilista_line=scilista_line):
filename = "/{}_{}.zip".format(*scilista_line.split())
mk_shutil.move.assert_any_call(
self.kwargs["xc_dir_name"] + filename, tmpdirname + filename
)
def test_get_sps_packages_must_return_packages_list_if_sps_packages_lst_exists(
self):
package_paths_list = [
str(pathlib.Path(self.proc_dir_name) / f"package_0{seq}.zip")
for seq in range(1, 4)
]
package_paths_list_file = pathlib.Path(
self.proc_dir_name) / "sps_packages.lst"
package_paths_list_file.write_text("\n".join(package_paths_list))
result = get_sps_packages(**self.kwargs)
self.assertEqual(result, package_paths_list)
def test_get_sps_packages_must_not_move_brag_if_there_is_ag_in_scilista(
self):
scilista_file_path = pathlib.Path(self.kwargs["scilista_file_path"])
package = "ag 2019nahead"
scilista_file_path.write_text(package)
source_filenames = (
"2020-05-22-10-00-34-480190_brag_2019nahead.zip\n"
"2020-05-22-10-00-34-480190_ag_2019nahead.zip").splitlines()
for filename in source_filenames:
zip_filename = pathlib.Path(self.xc_dir_name) / filename
with zipfile.ZipFile(zip_filename, "w") as zip_file:
zip_file.write(self.test_filepath)
result = get_sps_packages(**self.kwargs)
self.assertEqual([
str(
pathlib.Path(self.kwargs["proc_dir_name"]) /
"2020-05-22-10-00-34-480190_ag_2019nahead.zip")
], result)
def get_sps_packages(conf, **kwargs):
Logger.debug("create_all_subdags IN")
sps_packages = pre_sync_documents_to_kernel_operations.get_sps_packages(
Variable.get("SCILISTA_FILE_PATH"),
Variable.get("XC_SPS_PACKAGES_DIR"),
Variable.get("PROC_SPS_PACKAGES_DIR"),
)
for sps_package in sps_packages:
Logger.info("Triggering an external dag with package %s" % sps_package)
now = timezone.utcnow()
trigger_dag(
dag_id="sync_documents_to_kernel",
run_id="manual__%s_%s" %
(os.path.basename(sps_package), now.isoformat()),
execution_date=now,
replace_microseconds=False,
conf={"sps_package": sps_package},
)
Logger.debug("create_all_subdags OUT")
def test_read_scilista_from_file(self, mk_open):
get_sps_packages(**self.kwargs)
mk_open.assert_called_once_with("dir/path/scilista.lst")
def test_get_sps_packages_creates_path_dirs(self, mk_open, MockPath):
get_sps_packages(**self.kwargs)
MockPath.assert_any_call(self.kwargs["xc_dir_name"])
MockPath.assert_any_call(self.kwargs["proc_dir_name"])
