def test_invalid_proxy(self, proxy, exception, temp_file): with pytest.raises(exception): utilities.download_file( url='https://www.google.com', proxy=proxy, filename=temp_file, )
def test_snyk_windows(self, image, dockerfile): location = pathlib.Path(__file__).parent snyk_file = location / 'snyk.exe' download_file(SNYK_URL['windows'], snyk_file) dockerfile_args = [] if dockerfile: dockerfile_args.extend(['--file=', dockerfile]) process_clear = subprocess.run( [f'"{str(snyk_file)}"', 'config', 'clear'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) # nosec if process_clear.returncode != 0: pytest.fail('snyk clear config was failed') cmd_line = [ f'set SNYK_API={SNYK_API}&&', f'"{str(snyk_file)}"', 'auth', SNYK_TOKEN, '&&', f'"{str(snyk_file)}"', 'test', '--docker', image, *dockerfile_args ] process = subprocess.run(cmd_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) # nosec process_out = process.stdout.decode().replace(SNYK_TOKEN, '*' * 6) cmd_line[3] = '******' if process.returncode != 0: pytest.fail(f'SDL snyk issues: {cmd_line}\n{process_out}') else: print(f'SDL snyk output: {cmd_line}\n{process_out}')
def pytest_configure(config): config.addinivalue_line( 'markers', 'hddl: run tests on HDDL device', ) config.addinivalue_line( 'markers', 'vpu: run tests on VPU device', ) config.addinivalue_line( 'markers', 'gpu: run tests on GPU device', ) dist = config.getoption('--distribution') if dist in ('data_runtime', 'runtime', 'custom-no-omz', 'custom-no-cv'): log.info('Setting up runtime image dependencies') mount_root = pathlib.Path(config.getoption('--mount_root')) package_url = config.getoption('--package_url') image_os = config.getoption('--image_os') if (mount_root / 'openvino_dev').exists(): log.info( 'Directory for runtime testing dependency already exists, skipping dependency preparation' ) return if not package_url: return mount_root.mkdir(parents=True, exist_ok=True) dev_package_url = package_url.replace('_runtime_', '_dev_') # Temporarily, until there is no dev package for these distros if image_os in ('ubuntu20', 'centos7', 'centos8'): dev_package_url = dev_package_url.replace(image_os, 'ubuntu18') if package_url.startswith(('http://', 'https://', 'ftp://')): if 'win' in image_os: dldt_package = 'dldt.zip' else: dldt_package = 'dldt.tgz' log.info('Downloading dependent package...') download_file( dev_package_url, filename=mount_root / dldt_package, parents_=True, exist_ok_=True, ) log.info('Extracting dependent package...') unzip_file(str(mount_root / dldt_package), str(mount_root / 'openvino_dev')) log.info('Dependent package downloaded and extracted') else: dev_package_archive = pathlib.Path(dev_package_url) if dev_package_archive.exists(): unzip_file(str(dev_package_archive), str(mount_root / 'openvino_dev')) log.info('Dependent package extracted') else: err_msg = f"""Provided path of the dependent package should be an http/https/ftp access scheme or a local file in the project location as dependent package: {package_url}""" log.error(err_msg) raise FailedTest(err_msg)
def test_valid_url(self, temp_file): utilities.download_file( url='https://www.google.com/', filename=temp_file, ) if not temp_file.exists(): pytest.fail(f'{temp_file} not found') else: temp_file.unlink()
def build(self): """Building Docker image from dockerfile""" log.info(logger.LINE_DOUBLE) log.info('Preparing to build Docker image...') tmp_folder, self.args.old_package_url = '', '' if self.args.source == 'local' and self.args.package_url.startswith(('http://', 'https://', 'ftp://')): log.info('Downloading needed files...') self.args.old_package_url = self.args.package_url archive_name = self.args.old_package_url.split('/')[-1] tmp_folder = self.location / 'tmp' download_file(self.args.package_url, tmp_folder / archive_name, parents_=True) self.args.package_url = (tmp_folder / archive_name).relative_to(self.location) self.args.package_url = str(pathlib.PurePosixPath(self.args.package_url)) log.info('Downloading finished') self.kwargs['package_url'] = self.args.package_url log.info('Building Docker image...') self.args.file = pathlib.Path(self.args.file) if self.args.file.is_absolute(): self.args.file = pathlib.Path(self.args.file).relative_to(self.location) self.builder = DockerImageBuilder() curr_time = timeit.default_timer() if not self.args.openshift: self.kwargs.pop('openshift') log.info(f"Build log location: {self.logdir / 'image_build.log'}") self.image = self.builder.build_docker_image(dockerfile=self.args.file, directory=str(self.location), tag=self.image_name, build_args=self.kwargs, logfile=self.logdir / 'image_build.log') log.info(f'Build time: {format_timedelta(timeit.default_timer() - curr_time)}') if not self.image: raise FailedBuild(f'Error building Docker image {self.args.tags}') log.info(f'Save image data in {self.args.image_json_path} file') try: if not self.args.image_json_path.parent.exists(): self.args.image_json_path.parent.mkdir() with self.args.image_json_path.open(mode='w', encoding='utf-8') as f: json.dump({'image_name': self.image_name, 'product_version': self.args.product_version, 'distribution': self.args.distribution, 'os': self.args.os}, f, ensure_ascii=False, indent=4) except Exception: log.exception(f'Failed to save image data in {self.args.image_json_path} file') log.info(f'Docker image {self.args.tags} built successfully') if self.args.old_package_url: self.args.package_url, self.args.old_package_url = self.args.old_package_url, self.args.package_url self.kwargs['package_url'] = self.args.package_url if tmp_folder and tmp_folder.exists(): shutil.rmtree(tmp_folder, ignore_errors=True) log.info('Build dependencies deleted')
def test_valid_proxy(self, mock_session, temp_file): mock_session.return_value = mock.MagicMock(**{'get.side_effect': Exception('Proxy check')}) try: utilities.download_file( url='https://www.google.com/test', proxy={'https': 'https://www.google.com/test'}, filename=temp_file, ) except Exception as e: assert str(e) == 'Proxy check' # noqa: S101 # nosec
def test_dive_windows(self, image): location = pathlib.Path(__file__).parent dive_zip_file = location / 'dive.zip' download_file(DIVE_URL['windows'], dive_zip_file) unzip_file(str(dive_zip_file), str(location)) dive_file = location / 'dive.exe' cmd_line = [str(dive_file), '--ci', image] process = subprocess.run(cmd_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) # nosec if process.returncode != 0: pytest.fail(f'Linter dive issues: {process.stdout.decode()}') else: print(f'Linter dive output: {process.stdout.decode()}')
def download_equity_file() -> str: """ Downloads the Equity file from portal and extracts the ZIP Returns: Downloaded file name """ from datetime import timedelta final_file_name = None # Download the latest file available in last 4 days for i in range(5): # Get current filename and url now = datetime.now(tz=pytz.timezone("Asia/Kolkata")) - timedelta( days=i) today = f"{now.strftime('%d')}{now.strftime('%m')}{now.strftime('%y')}" filename = FILE_NAME.format(today) download_link = FILE_URL.format(filename=filename) # Try downloading file try: if download_file(download_link, f"{filename}.zip", "/tmp"): # File download is successful, break final_file_name = filename break except ErrorDownloadingFile: pass # If unable to download the file raise Exception if not final_file_name: raise ErrorDownloadingFile("Failed") # If download successful extract the ZIP import zipfile with zipfile.ZipFile(f"/tmp/{final_file_name}.zip", "r") as zip_ref: zip_ref.extractall("/tmp/") # Return the csv file path after extract return f"/tmp/{final_file_name}.CSV"
def test_invalid_url(self, url, exception, temp_file): with pytest.raises(exception): utilities.download_file( url=url, filename=temp_file, )