def _download_pypi_sdk_package(temp_dir): """Downloads SDK package from PyPI and returns path to local path.""" # TODO(silviuc): Handle apache-beam versions when we have official releases. import pkg_resources as pkg try: version = pkg.get_distribution(GOOGLE_PACKAGE_NAME).version except pkg.DistributionNotFound: raise RuntimeError('Please set --sdk_location command-line option ' 'or install a valid {} distribution.' .format(GOOGLE_PACKAGE_NAME)) # Get a source distribution for the SDK package from PyPI. cmd_args = [ _get_python_executable(), '-m', 'pip', 'install', '--download', temp_dir, '%s==%s' % (GOOGLE_PACKAGE_NAME, version), '--no-binary', ':all:', '--no-deps'] logging.info('Executing command: %s', cmd_args) processes.check_call(cmd_args) zip_expected = os.path.join( temp_dir, '%s-%s.zip' % (GOOGLE_PACKAGE_NAME, version)) if os.path.exists(zip_expected): return zip_expected tgz_expected = os.path.join( temp_dir, '%s-%s.tar.gz' % (GOOGLE_PACKAGE_NAME, version)) if os.path.exists(tgz_expected): return tgz_expected raise RuntimeError( 'Failed to download a source distribution for the running SDK. Expected ' 'either %s or %s to be found in the download folder.' % ( zip_expected, tgz_expected))
def test_method_forwarding_windows(self, *unused_mocks): # Test that the correct calls are being forwarded to the subprocess module # and that the shell=True flag is added when we are on Windows. processes.force_shell = True processes.call(['subprocess', 'call'], shell=False, other_arg=True) processes.subprocess.call.assert_called_once_with( ['subprocess', 'call'], shell=True, other_arg=True) processes.check_call( ['subprocess', 'check_call'], shell=False, other_arg=True) processes.subprocess.check_call.assert_called_once_with( ['subprocess', 'check_call'], shell=True, other_arg=True) processes.check_output( ['subprocess', 'check_output'], shell=False, other_arg=True) processes.subprocess.check_output.assert_called_once_with( ['subprocess', 'check_output'], shell=True, other_arg=True) processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True) processes.subprocess.Popen.assert_called_once_with( ['subprocess', 'Popen'], shell=True, other_arg=True)
def _download_pypi_sdk_package(temp_dir): """Downloads SDK package from PyPI and returns path to local path.""" package_name = get_sdk_package_name() try: version = pkg_resources.get_distribution(package_name).version except pkg_resources.DistributionNotFound: raise RuntimeError('Please set --sdk_location command-line option ' 'or install a valid {} distribution.' .format(package_name)) # Get a source distribution for the SDK package from PyPI. cmd_args = [ _get_python_executable(), '-m', 'pip', 'download', '--dest', temp_dir, '%s==%s' % (package_name, version), '--no-binary', ':all:', '--no-deps'] logging.info('Executing command: %s', cmd_args) processes.check_call(cmd_args) zip_expected = os.path.join( temp_dir, '%s-%s.zip' % (package_name, version)) if os.path.exists(zip_expected): return zip_expected tgz_expected = os.path.join( temp_dir, '%s-%s.tar.gz' % (package_name, version)) if os.path.exists(tgz_expected): return tgz_expected raise RuntimeError( 'Failed to download a source distribution for the running SDK. Expected ' 'either %s or %s to be found in the download folder.' % ( zip_expected, tgz_expected))
def _download_pypi_sdk_package(temp_dir, fetch_binary=False, language_version_tag='27', language_implementation_tag='cp', abi_tag='cp27mu', platform_tag='manylinux1_x86_64'): """Downloads SDK package from PyPI and returns path to local path.""" package_name = Stager.get_sdk_package_name() try: version = pkg_resources.get_distribution(package_name).version except pkg_resources.DistributionNotFound: raise RuntimeError('Please set --sdk_location command-line option ' 'or install a valid {} distribution.' .format(package_name)) cmd_args = [ Stager._get_python_executable(), '-m', 'pip', 'download', '--dest', temp_dir, '%s==%s' % (package_name, version), '--no-deps' ] if fetch_binary: logging.info('Downloading binary distribtution of the SDK from PyPi') # Get a wheel distribution for the SDK from PyPI. cmd_args.extend([ '--only-binary', ':all:', '--python-version', language_version_tag, '--implementation', language_implementation_tag, '--abi', abi_tag, '--platform', platform_tag ]) # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl expected_files = [ os.path.join( temp_dir, '%s-%s-%s%s-%s-%s.whl' % (package_name.replace( '-', '_'), version, language_implementation_tag, language_version_tag, abi_tag, platform_tag)) ] else: logging.info('Downloading source distribtution of the SDK from PyPi') cmd_args.extend(['--no-binary', ':all:']) expected_files = [ os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)), os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version)) ] logging.info('Executing command: %s', cmd_args) try: processes.check_call(cmd_args) except subprocess.CalledProcessError as e: raise RuntimeError(repr(e)) for sdk_file in expected_files: if os.path.exists(sdk_file): return sdk_file raise RuntimeError( 'Failed to download a distribution for the running SDK. ' 'Expected either one of %s to be found in the download folder.' % (expected_files))
def test_oserror_check_call_message(self): # Configure the mock to return a response with an OK status code. self.mock_get.side_effect = OSError() cmd = ["lls"] try: processes.check_call(cmd) except RuntimeError as error: self.assertIn('Executable {} not found'.format(str(cmd)),\ error.args[0])
def _populate_requirements_cache(requirements_file, cache_dir): # The 'pip download' command will not download again if it finds the # tarball with the proper version already present. # It will get the packages downloaded in the order they are presented in # the requirements file and will not download package dependencies. cmd_args = [ _get_python_executable(), '-m', 'pip', 'install', '--download', cache_dir, '-r', requirements_file, # Download from PyPI source distributions. '--no-binary', ':all:'] logging.info('Executing command: %s', cmd_args) processes.check_call(cmd_args)
def _dependency_file_copy(from_path, to_path): """Copies a local file to a GCS file or vice versa.""" logging.info('file copy from %s to %s.', from_path, to_path) if from_path.startswith('gs://') or to_path.startswith('gs://'): command_args = ['gsutil', '-m', '-q', 'cp', from_path, to_path] logging.info('Executing command: %s', command_args) processes.check_call(command_args) else: # Branch used only for unit tests and integration tests. # In such environments GCS support is not available. if not os.path.isdir(os.path.dirname(to_path)): logging.info('Created folder (since we have not done yet, and any errors ' 'will follow): %s ', os.path.dirname(to_path)) os.mkdir(os.path.dirname(to_path)) shutil.copyfile(from_path, to_path)
def _build_setup_package(setup_file, temp_dir, build_setup_args=None): saved_current_directory = os.getcwd() try: os.chdir(os.path.dirname(setup_file)) if build_setup_args is None: build_setup_args = [ _get_python_executable(), os.path.basename(setup_file), 'sdist', '--dist-dir', temp_dir] logging.info('Executing command: %s', build_setup_args) processes.check_call(build_setup_args) output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz')) if not output_files: raise RuntimeError( 'File %s not found.' % os.path.join(temp_dir, '*.tar.gz')) return output_files[0] finally: os.chdir(saved_current_directory)
def test_method_forwarding_windows(self, *unused_mocks): # Test that the correct calls are being forwarded to the subprocess module # and that the shell=True flag is added when we are on Windows. processes.force_shell = True processes.call(['subprocess', 'call'], shell=False, other_arg=True) processes.subprocess.call.assert_called_once_with( ['subprocess', 'call'], shell=True, other_arg=True) processes.check_call(['subprocess', 'check_call'], shell=False, other_arg=True) processes.subprocess.check_call.assert_called_once_with( ['subprocess', 'check_call'], shell=True, other_arg=True) processes.check_output(['subprocess', 'check_output'], shell=False) processes.subprocess.check_output.assert_called_once_with( ['subprocess', 'check_output'], shell=True) processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True) processes.subprocess.Popen.assert_called_once_with( ['subprocess', 'Popen'], shell=True, other_arg=True)
def test_check_call_pip_install_non_existing_package(self): returncode = 1 package = "non-exsisting-package" cmd = ['python', '-m', 'pip', 'download', '--dest', '/var',\ '{}'.format(package),\ '--no-deps', '--no-binary', ':all:'] output = "Collecting {}".format(package) self.mock_get.side_effect = subprocess.CalledProcessError(returncode,\ cmd, output=output) try: output = processes.check_call(cmd) self.fail("The test failed due to that\ no error was raised when calling process.check_call") except RuntimeError as error: self.assertIn("Output from execution of subprocess: {}".format(output),\ error.args[0]) self.assertIn("Pip install failed for package: {}".format(package),\ error.args[0])
def test_oserror_check_call(self): # Configure the mock to return a response with an OK status code. self.mock_get.side_effect = OSError("Test OSError") with self.assertRaises(RuntimeError): processes.check_call(["lls"])
def _download_pypi_sdk_package(temp_dir, fetch_binary=False, language_version_tag='27', language_implementation_tag='cp', abi_tag='cp27mu', platform_tag='manylinux1_x86_64'): """Downloads SDK package from PyPI and returns path to local path.""" package_name = Stager.get_sdk_package_name() try: version = pkg_resources.get_distribution(package_name).version except pkg_resources.DistributionNotFound: raise RuntimeError( 'Please set --sdk_location command-line option ' 'or install a valid {} distribution.'.format(package_name)) cmd_args = [ Stager._get_python_executable(), '-m', 'pip', 'download', '--dest', temp_dir, '%s==%s' % (package_name, version), '--no-deps' ] if fetch_binary: logging.info( 'Downloading binary distribtution of the SDK from PyPi') # Get a wheel distribution for the SDK from PyPI. cmd_args.extend([ '--only-binary', ':all:', '--python-version', language_version_tag, '--implementation', language_implementation_tag, '--abi', abi_tag, '--platform', platform_tag ]) # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl expected_files = [ os.path.join( temp_dir, '%s-%s-%s%s-%s-%s.whl' % (package_name.replace( '-', '_'), version, language_implementation_tag, language_version_tag, abi_tag, platform_tag)) ] else: logging.info( 'Downloading source distribtution of the SDK from PyPi') cmd_args.extend(['--no-binary', ':all:']) expected_files = [ os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)), os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version)) ] logging.info('Executing command: %s', cmd_args) try: processes.check_call(cmd_args) except subprocess.CalledProcessError as e: raise RuntimeError(repr(e)) for sdk_file in expected_files: if os.path.exists(sdk_file): return sdk_file raise RuntimeError( 'Failed to download a distribution for the running SDK. ' 'Expected either one of %s to be found in the download folder.' % (expected_files))