def test_generate_pinfile(self):
# given
req_in = self.cwd / "req.in"
req_txt = self.cwd / "req.txt"
pin_in = self.cwd / "pin.in"
req_in.write_text("""
requests==2.25.1
""")
bf_pip.pip_compile(req_in)
# when
with self.assertLogs(level=logging.WARNING):
bf_pip.generate_pinfile(
req_in,
pin_in,
lambda: ["idna==2.0", "chardet==4", "urllib3~=1.26,<1.26.3"],
)
# then
self.assertFileContentRegex(pin_in, r"(?m)^## idna==2.0 +")
self.assertFileContentRegex(pin_in, r"(?m)^urllib3~=1.26,<1.26.3$")
self.assertFileContentRegex(pin_in, r"(?m)^chardet==4$")
self.assertFileContentRegex(req_txt, r"(?m)^requests==2.25.1$")
self.assertFileContentRegex(req_txt, r"(?m)^idna==2.10$")
self.assertFileContentRegex(req_txt, r"(?m)^urllib3==1.26.2$")
self.assertFileContentRegex(req_txt, r"(?m)^chardet==4.0.0$")
def start_project(
is_basic,
project_name,
projects_id,
composers_bucket,
envs,
pyspark_job,
):
project_dir = project_name + '_project'
project_path = Path(project_dir).resolve()
render_builtin_templates(
project_path,
"new-project",
variables={
'is_basic': is_basic,
'project_id': projects_id[0],
'dags_bucket': composers_bucket[0],
'bigflow_version': bigflow.__version__,
'project_name': project_name,
'envs': envs,
'composers_bucket': composers_bucket,
pyspark_job: True,
},
)
import bigflow.build.pip as bf_pip
bf_pip.pip_compile(project_path / "resources" / "requirements.txt")
def create_dirs_and_files(config, templates):
project_dir = config['project_name'] + '_project'
project_path = Path(project_dir).resolve()
workflows_path = Path(project_dir) / config['project_name']
word_count_path = workflows_path / 'wordcount'
internationalports_path = workflows_path / 'internationalports'
test_path = Path(project_dir + '/test')
os.mkdir(project_path)
os.mkdir(workflows_path)
os.mkdir(word_count_path)
os.mkdir(internationalports_path)
os.mkdir(test_path)
create_module_files(templates['main_templates'], project_path.resolve())
create_module_files(templates['beam_templates'], word_count_path.resolve())
create_module_files(templates['bq_templates'],
internationalports_path.resolve())
create_module_files(templates['test_templates'], test_path.resolve())
create_module_files({'__init__.py': ''}, workflows_path.resolve())
render_builtin_templates(
project_path,
"new-project",
variables={
'project_id': config['projects_id'][0],
'bigflow_version': bigflow.__version__,
**config,
},
)
import bigflow.build.pip as bf_pip
bf_pip.pip_compile(project_path / "resources" / "requirements.txt")
def test_check_dataflow_conflicts(self):
# given
self.req_in.write_text("""
apache-beam==2.25
freezegun==0.3.14
""")
bf_pip.pip_compile(self.req_in)
# then
with self.assertLogs(dc.logger, level=logging.ERROR):
dc.check_beam_worker_dependencies_conflict(self.req_in)
def test_should_compile_requirements(self):
# given
req_in = self.cwd / "req.in"
req_txt = self.cwd / "req.txt"
req_in.write_text("pandas>=1.1")
# when
bf_pip.pip_compile(req_in)
# then
reqs = req_txt.read_text()
self.assertIn("pandas==", reqs)
self.assertIn("", reqs)
def test_should_ignore_pip_flags(self):
# given
req_in = self.cwd / "req.in"
req_txt = self.cwd / "req.txt"
req_in.write_text("""
--extra-index-url https://example.org/python/repo
pandas
""")
# when
bf_pip.pip_compile(req_in)
# then
reqs = req_txt.read_text()
self.assertIn("pandas==", reqs)
self.assertIn("", reqs)
def test_generate_dataflow_pinfile(self, read_setuppy_args,
detect_py_version):
# given
detect_py_version.return_value = "3.7"
read_setuppy_args.return_value = {
'project_requirements_file': str(self.req_txt)
}
self.req_in.write_text("""
# see `beam2.25_py3.7.txt`
fastavro==0.24.2
typing-extensions==3.7.4.2
grpcio==1.30.0
httplib2==0.17.3
pymongo==3.10.1
requests==2.24.0
protobuf==3.12.2
chardet==3.0.4
pbr==5.5.0
avro-python3==1.8.2
urllib3==1.25.10
certifi==2020.6.20
oauth2client==3.0.0
numpy==1.18.4
#pytz==2020.1
#rsa==4.6
apache-beam==2.25
rsa==4.7 # unersolvable conflict
pytz # latest version, resolvable conflict
""")
bf_pip.pip_compile(self.req_in)
with self.assertLogs(level=logging.WARNING) as logs:
dc.sync_requirements_with_dataflow_workers()
# then
self.assertFileExists(self.pins_in)
self.assertFileContentRegex(self.req_in, r"(?mi)^-r dataflow_pins.in")
self.assertFileContentRegex(self.pins_in, r"(?m)^## rsa==4.6")
self.assertFileContentRegex(self.pins_in, r"(?m)^pytz==2020.1$")
logs = "\n".join(logs.output)
self.assertRegex(logs, r"Failed to pin some libraries")
self.assertRegex(logs, r"\Wrsa\W")
def test_detect_when_requirements_was_changed(self):
# given
req_in = self.cwd / "req.in"
req_in.write_text("pandas>=1.1")
# when
bf_pip.pip_compile(req_in)
# then
self.assertFalse(bf_pip.check_requirements_needs_recompile(req_in))
# when
req_in.write_text("pandas>=1.1.1,<2")
# then
with self.assertLogs(bf_pip.logger, level=logging.WARNING):
self.assertTrue(bf_pip.check_requirements_needs_recompile(req_in))
def test_detect_dataflow_conflicts(self, detect_py_version):
# given
detect_py_version.return_value = "3.7"
self.req_in.write_text("""
apache-beam==2.25
freezegun==0.3.14
fastavro==1.2.0
requests==2.25.0
""")
bf_pip.pip_compile(self.req_in)
# when
conflicts = dc.detect_dataflow_conflicts(self.req_in)
# then
self.assertEqual(conflicts.get('freezegun'), ("0.3.14", "0.3.15"))
self.assertEqual(conflicts.get('fastavro'), ("1.2.0", "0.24.2"))
self.assertEqual(conflicts.get('requests'), ("2.25.0", "2.24.0"))