def test_invalid_proxy(self, proxy, exception, temp_file):
with pytest.raises(exception):
utilities.download_file(
url='https://www.google.com',
proxy=proxy,
filename=temp_file,
)
def test_snyk_windows(self, image, dockerfile):
location = pathlib.Path(__file__).parent
snyk_file = location / 'snyk.exe'
download_file(SNYK_URL['windows'], snyk_file)
dockerfile_args = []
if dockerfile:
dockerfile_args.extend(['--file=', dockerfile])
process_clear = subprocess.run(
[f'"{str(snyk_file)}"', 'config', 'clear'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False) # nosec
if process_clear.returncode != 0:
pytest.fail('snyk clear config was failed')
cmd_line = [
f'set SNYK_API={SNYK_API}&&', f'"{str(snyk_file)}"', 'auth',
SNYK_TOKEN, '&&', f'"{str(snyk_file)}"', 'test', '--docker', image,
*dockerfile_args
]
process = subprocess.run(cmd_line,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False) # nosec
process_out = process.stdout.decode().replace(SNYK_TOKEN, '*' * 6)
cmd_line[3] = '******'
if process.returncode != 0:
pytest.fail(f'SDL snyk issues: {cmd_line}\n{process_out}')
else:
print(f'SDL snyk output: {cmd_line}\n{process_out}')
def pytest_configure(config):
config.addinivalue_line(
'markers',
'hddl: run tests on HDDL device',
)
config.addinivalue_line(
'markers',
'vpu: run tests on VPU device',
)
config.addinivalue_line(
'markers',
'gpu: run tests on GPU device',
)
dist = config.getoption('--distribution')
if dist in ('data_runtime', 'runtime', 'custom-no-omz', 'custom-no-cv'):
log.info('Setting up runtime image dependencies')
mount_root = pathlib.Path(config.getoption('--mount_root'))
package_url = config.getoption('--package_url')
image_os = config.getoption('--image_os')
if (mount_root / 'openvino_dev').exists():
log.info(
'Directory for runtime testing dependency already exists, skipping dependency preparation'
)
return
if not package_url:
return
mount_root.mkdir(parents=True, exist_ok=True)
dev_package_url = package_url.replace('_runtime_', '_dev_')
# Temporarily, until there is no dev package for these distros
if image_os in ('ubuntu20', 'centos7', 'centos8'):
dev_package_url = dev_package_url.replace(image_os, 'ubuntu18')
if package_url.startswith(('http://', 'https://', 'ftp://')):
if 'win' in image_os:
dldt_package = 'dldt.zip'
else:
dldt_package = 'dldt.tgz'
log.info('Downloading dependent package...')
download_file(
dev_package_url,
filename=mount_root / dldt_package,
parents_=True,
exist_ok_=True,
)
log.info('Extracting dependent package...')
unzip_file(str(mount_root / dldt_package),
str(mount_root / 'openvino_dev'))
log.info('Dependent package downloaded and extracted')
else:
dev_package_archive = pathlib.Path(dev_package_url)
if dev_package_archive.exists():
unzip_file(str(dev_package_archive),
str(mount_root / 'openvino_dev'))
log.info('Dependent package extracted')
else:
err_msg = f"""Provided path of the dependent package should be an http/https/ftp access scheme
or a local file in the project location as dependent package: {package_url}"""
log.error(err_msg)
raise FailedTest(err_msg)
def test_valid_url(self, temp_file):
utilities.download_file(
url='https://www.google.com/',
filename=temp_file,
)
if not temp_file.exists():
pytest.fail(f'{temp_file} not found')
else:
temp_file.unlink()
def build(self):
"""Building Docker image from dockerfile"""
log.info(logger.LINE_DOUBLE)
log.info('Preparing to build Docker image...')
tmp_folder, self.args.old_package_url = '', ''
if self.args.source == 'local' and self.args.package_url.startswith(('http://', 'https://', 'ftp://')):
log.info('Downloading needed files...')
self.args.old_package_url = self.args.package_url
archive_name = self.args.old_package_url.split('/')[-1]
tmp_folder = self.location / 'tmp'
download_file(self.args.package_url, tmp_folder / archive_name, parents_=True)
self.args.package_url = (tmp_folder / archive_name).relative_to(self.location)
self.args.package_url = str(pathlib.PurePosixPath(self.args.package_url))
log.info('Downloading finished')
self.kwargs['package_url'] = self.args.package_url
log.info('Building Docker image...')
self.args.file = pathlib.Path(self.args.file)
if self.args.file.is_absolute():
self.args.file = pathlib.Path(self.args.file).relative_to(self.location)
self.builder = DockerImageBuilder()
curr_time = timeit.default_timer()
if not self.args.openshift:
self.kwargs.pop('openshift')
log.info(f"Build log location: {self.logdir / 'image_build.log'}")
self.image = self.builder.build_docker_image(dockerfile=self.args.file,
directory=str(self.location),
tag=self.image_name,
build_args=self.kwargs,
logfile=self.logdir / 'image_build.log')
log.info(f'Build time: {format_timedelta(timeit.default_timer() - curr_time)}')
if not self.image:
raise FailedBuild(f'Error building Docker image {self.args.tags}')
log.info(f'Save image data in {self.args.image_json_path} file')
try:
if not self.args.image_json_path.parent.exists():
self.args.image_json_path.parent.mkdir()
with self.args.image_json_path.open(mode='w', encoding='utf-8') as f:
json.dump({'image_name': self.image_name,
'product_version': self.args.product_version,
'distribution': self.args.distribution,
'os': self.args.os}, f, ensure_ascii=False, indent=4)
except Exception:
log.exception(f'Failed to save image data in {self.args.image_json_path} file')
log.info(f'Docker image {self.args.tags} built successfully')
if self.args.old_package_url:
self.args.package_url, self.args.old_package_url = self.args.old_package_url, self.args.package_url
self.kwargs['package_url'] = self.args.package_url
if tmp_folder and tmp_folder.exists():
shutil.rmtree(tmp_folder, ignore_errors=True)
log.info('Build dependencies deleted')
def test_valid_proxy(self, mock_session, temp_file):
mock_session.return_value = mock.MagicMock(**{'get.side_effect': Exception('Proxy check')})
try:
utilities.download_file(
url='https://www.google.com/test',
proxy={'https': 'https://www.google.com/test'},
filename=temp_file,
)
except Exception as e:
assert str(e) == 'Proxy check' # noqa: S101 # nosec
def test_dive_windows(self, image):
location = pathlib.Path(__file__).parent
dive_zip_file = location / 'dive.zip'
download_file(DIVE_URL['windows'], dive_zip_file)
unzip_file(str(dive_zip_file), str(location))
dive_file = location / 'dive.exe'
cmd_line = [str(dive_file), '--ci', image]
process = subprocess.run(cmd_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) # nosec
if process.returncode != 0:
pytest.fail(f'Linter dive issues: {process.stdout.decode()}')
else:
print(f'Linter dive output: {process.stdout.decode()}')
def download_equity_file() -> str:
"""
Downloads the Equity file from portal and extracts the ZIP
Returns:
Downloaded file name
"""
from datetime import timedelta
final_file_name = None
# Download the latest file available in last 4 days
for i in range(5):
# Get current filename and url
now = datetime.now(tz=pytz.timezone("Asia/Kolkata")) - timedelta(
days=i)
today = f"{now.strftime('%d')}{now.strftime('%m')}{now.strftime('%y')}"
filename = FILE_NAME.format(today)
download_link = FILE_URL.format(filename=filename)
# Try downloading file
try:
if download_file(download_link, f"{filename}.zip", "/tmp"):
# File download is successful, break
final_file_name = filename
break
except ErrorDownloadingFile:
pass
# If unable to download the file raise Exception
if not final_file_name:
raise ErrorDownloadingFile("Failed")
# If download successful extract the ZIP
import zipfile
with zipfile.ZipFile(f"/tmp/{final_file_name}.zip", "r") as zip_ref:
zip_ref.extractall("/tmp/")
# Return the csv file path after extract
return f"/tmp/{final_file_name}.CSV"
def test_invalid_url(self, url, exception, temp_file):
with pytest.raises(exception):
utilities.download_file(
url=url,
filename=temp_file,
)
