def _download_pypi_sdk_package(temp_dir):
"""Downloads SDK package from PyPI and returns path to local path."""
# TODO(silviuc): Handle apache-beam versions when we have official releases.
import pkg_resources as pkg
try:
version = pkg.get_distribution(GOOGLE_PACKAGE_NAME).version
except pkg.DistributionNotFound:
raise RuntimeError('Please set --sdk_location command-line option '
'or install a valid {} distribution.'
.format(GOOGLE_PACKAGE_NAME))
# Get a source distribution for the SDK package from PyPI.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'install', '--download', temp_dir,
'%s==%s' % (GOOGLE_PACKAGE_NAME, version),
'--no-binary', ':all:', '--no-deps']
logging.info('Executing command: %s', cmd_args)
processes.check_call(cmd_args)
zip_expected = os.path.join(
temp_dir, '%s-%s.zip' % (GOOGLE_PACKAGE_NAME, version))
if os.path.exists(zip_expected):
return zip_expected
tgz_expected = os.path.join(
temp_dir, '%s-%s.tar.gz' % (GOOGLE_PACKAGE_NAME, version))
if os.path.exists(tgz_expected):
return tgz_expected
raise RuntimeError(
'Failed to download a source distribution for the running SDK. Expected '
'either %s or %s to be found in the download folder.' % (
zip_expected, tgz_expected))
def test_method_forwarding_windows(self, *unused_mocks):
# Test that the correct calls are being forwarded to the subprocess module
# and that the shell=True flag is added when we are on Windows.
processes.force_shell = True
processes.call(['subprocess', 'call'], shell=False, other_arg=True)
processes.subprocess.call.assert_called_once_with(
['subprocess', 'call'],
shell=True,
other_arg=True)
processes.check_call(
['subprocess', 'check_call'],
shell=False,
other_arg=True)
processes.subprocess.check_call.assert_called_once_with(
['subprocess', 'check_call'],
shell=True,
other_arg=True)
processes.check_output(
['subprocess', 'check_output'],
shell=False,
other_arg=True)
processes.subprocess.check_output.assert_called_once_with(
['subprocess', 'check_output'],
shell=True,
other_arg=True)
processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True)
processes.subprocess.Popen.assert_called_once_with(
['subprocess', 'Popen'],
shell=True,
other_arg=True)
def _download_pypi_sdk_package(temp_dir):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = get_sdk_package_name()
try:
version = pkg_resources.get_distribution(package_name).version
except pkg_resources.DistributionNotFound:
raise RuntimeError('Please set --sdk_location command-line option '
'or install a valid {} distribution.'
.format(package_name))
# Get a source distribution for the SDK package from PyPI.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'download', '--dest', temp_dir,
'%s==%s' % (package_name, version),
'--no-binary', ':all:', '--no-deps']
logging.info('Executing command: %s', cmd_args)
processes.check_call(cmd_args)
zip_expected = os.path.join(
temp_dir, '%s-%s.zip' % (package_name, version))
if os.path.exists(zip_expected):
return zip_expected
tgz_expected = os.path.join(
temp_dir, '%s-%s.tar.gz' % (package_name, version))
if os.path.exists(tgz_expected):
return tgz_expected
raise RuntimeError(
'Failed to download a source distribution for the running SDK. Expected '
'either %s or %s to be found in the download folder.' % (
zip_expected, tgz_expected))
def _download_pypi_sdk_package(temp_dir,
fetch_binary=False,
language_version_tag='27',
language_implementation_tag='cp',
abi_tag='cp27mu',
platform_tag='manylinux1_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
version = pkg_resources.get_distribution(package_name).version
except pkg_resources.DistributionNotFound:
raise RuntimeError('Please set --sdk_location command-line option '
'or install a valid {} distribution.'
.format(package_name))
cmd_args = [
Stager._get_python_executable(), '-m', 'pip', 'download', '--dest',
temp_dir,
'%s==%s' % (package_name, version), '--no-deps'
]
if fetch_binary:
logging.info('Downloading binary distribtution of the SDK from PyPi')
# Get a wheel distribution for the SDK from PyPI.
cmd_args.extend([
'--only-binary', ':all:', '--python-version', language_version_tag,
'--implementation', language_implementation_tag, '--abi', abi_tag,
'--platform', platform_tag
])
# Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
expected_files = [
os.path.join(
temp_dir, '%s-%s-%s%s-%s-%s.whl' % (package_name.replace(
'-', '_'), version, language_implementation_tag,
language_version_tag, abi_tag,
platform_tag))
]
else:
logging.info('Downloading source distribtution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
expected_files = [
os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)),
os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version))
]
logging.info('Executing command: %s', cmd_args)
try:
processes.check_call(cmd_args)
except subprocess.CalledProcessError as e:
raise RuntimeError(repr(e))
for sdk_file in expected_files:
if os.path.exists(sdk_file):
return sdk_file
raise RuntimeError(
'Failed to download a distribution for the running SDK. '
'Expected either one of %s to be found in the download folder.' %
(expected_files))
def test_oserror_check_call_message(self):
# Configure the mock to return a response with an OK status code.
self.mock_get.side_effect = OSError()
cmd = ["lls"]
try:
processes.check_call(cmd)
except RuntimeError as error:
self.assertIn('Executable {} not found'.format(str(cmd)),\
error.args[0])
def _populate_requirements_cache(requirements_file, cache_dir):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will not download package dependencies.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'install', '--download', cache_dir,
'-r', requirements_file,
# Download from PyPI source distributions.
'--no-binary', ':all:']
logging.info('Executing command: %s', cmd_args)
processes.check_call(cmd_args)
def _populate_requirements_cache(requirements_file, cache_dir):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will not download package dependencies.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'install', '--download', cache_dir,
'-r', requirements_file,
# Download from PyPI source distributions.
'--no-binary', ':all:']
logging.info('Executing command: %s', cmd_args)
processes.check_call(cmd_args)
def _dependency_file_copy(from_path, to_path):
"""Copies a local file to a GCS file or vice versa."""
logging.info('file copy from %s to %s.', from_path, to_path)
if from_path.startswith('gs://') or to_path.startswith('gs://'):
command_args = ['gsutil', '-m', '-q', 'cp', from_path, to_path]
logging.info('Executing command: %s', command_args)
processes.check_call(command_args)
else:
# Branch used only for unit tests and integration tests.
# In such environments GCS support is not available.
if not os.path.isdir(os.path.dirname(to_path)):
logging.info('Created folder (since we have not done yet, and any errors '
'will follow): %s ', os.path.dirname(to_path))
os.mkdir(os.path.dirname(to_path))
shutil.copyfile(from_path, to_path)
def _dependency_file_copy(from_path, to_path):
"""Copies a local file to a GCS file or vice versa."""
logging.info('file copy from %s to %s.', from_path, to_path)
if from_path.startswith('gs://') or to_path.startswith('gs://'):
command_args = ['gsutil', '-m', '-q', 'cp', from_path, to_path]
logging.info('Executing command: %s', command_args)
processes.check_call(command_args)
else:
# Branch used only for unit tests and integration tests.
# In such environments GCS support is not available.
if not os.path.isdir(os.path.dirname(to_path)):
logging.info('Created folder (since we have not done yet, and any errors '
'will follow): %s ', os.path.dirname(to_path))
os.mkdir(os.path.dirname(to_path))
shutil.copyfile(from_path, to_path)
def _build_setup_package(setup_file, temp_dir, build_setup_args=None):
saved_current_directory = os.getcwd()
try:
os.chdir(os.path.dirname(setup_file))
if build_setup_args is None:
build_setup_args = [
_get_python_executable(), os.path.basename(setup_file),
'sdist', '--dist-dir', temp_dir]
logging.info('Executing command: %s', build_setup_args)
processes.check_call(build_setup_args)
output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz'))
if not output_files:
raise RuntimeError(
'File %s not found.' % os.path.join(temp_dir, '*.tar.gz'))
return output_files[0]
finally:
os.chdir(saved_current_directory)
def _build_setup_package(setup_file, temp_dir, build_setup_args=None):
saved_current_directory = os.getcwd()
try:
os.chdir(os.path.dirname(setup_file))
if build_setup_args is None:
build_setup_args = [
_get_python_executable(), os.path.basename(setup_file),
'sdist', '--dist-dir', temp_dir]
logging.info('Executing command: %s', build_setup_args)
processes.check_call(build_setup_args)
output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz'))
if not output_files:
raise RuntimeError(
'File %s not found.' % os.path.join(temp_dir, '*.tar.gz'))
return output_files[0]
finally:
os.chdir(saved_current_directory)
def test_method_forwarding_windows(self, *unused_mocks):
# Test that the correct calls are being forwarded to the subprocess module
# and that the shell=True flag is added when we are on Windows.
processes.force_shell = True
processes.call(['subprocess', 'call'], shell=False, other_arg=True)
processes.subprocess.call.assert_called_once_with(
['subprocess', 'call'], shell=True, other_arg=True)
processes.check_call(['subprocess', 'check_call'],
shell=False,
other_arg=True)
processes.subprocess.check_call.assert_called_once_with(
['subprocess', 'check_call'], shell=True, other_arg=True)
processes.check_output(['subprocess', 'check_output'], shell=False)
processes.subprocess.check_output.assert_called_once_with(
['subprocess', 'check_output'], shell=True)
processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True)
processes.subprocess.Popen.assert_called_once_with(
['subprocess', 'Popen'], shell=True, other_arg=True)
def test_check_call_pip_install_non_existing_package(self):
returncode = 1
package = "non-exsisting-package"
cmd = ['python', '-m', 'pip', 'download', '--dest', '/var',\
'{}'.format(package),\
'--no-deps', '--no-binary', ':all:']
output = "Collecting {}".format(package)
self.mock_get.side_effect = subprocess.CalledProcessError(returncode,\
cmd, output=output)
try:
output = processes.check_call(cmd)
self.fail("The test failed due to that\
no error was raised when calling process.check_call")
except RuntimeError as error:
self.assertIn("Output from execution of subprocess: {}".format(output),\
error.args[0])
self.assertIn("Pip install failed for package: {}".format(package),\
error.args[0])
def test_check_call_pip_install_non_existing_package(self):
returncode = 1
package = "non-exsisting-package"
cmd = ['python', '-m', 'pip', 'download', '--dest', '/var',\
'{}'.format(package),\
'--no-deps', '--no-binary', ':all:']
output = "Collecting {}".format(package)
self.mock_get.side_effect = subprocess.CalledProcessError(returncode,\
cmd, output=output)
try:
output = processes.check_call(cmd)
self.fail("The test failed due to that\
no error was raised when calling process.check_call")
except RuntimeError as error:
self.assertIn("Output from execution of subprocess: {}".format(output),\
error.args[0])
self.assertIn("Pip install failed for package: {}".format(package),\
error.args[0])
def test_oserror_check_call(self):
# Configure the mock to return a response with an OK status code.
self.mock_get.side_effect = OSError("Test OSError")
with self.assertRaises(RuntimeError):
processes.check_call(["lls"])
def _download_pypi_sdk_package(temp_dir,
fetch_binary=False,
language_version_tag='27',
language_implementation_tag='cp',
abi_tag='cp27mu',
platform_tag='manylinux1_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
version = pkg_resources.get_distribution(package_name).version
except pkg_resources.DistributionNotFound:
raise RuntimeError(
'Please set --sdk_location command-line option '
'or install a valid {} distribution.'.format(package_name))
cmd_args = [
Stager._get_python_executable(), '-m', 'pip', 'download', '--dest',
temp_dir,
'%s==%s' % (package_name, version), '--no-deps'
]
if fetch_binary:
logging.info(
'Downloading binary distribtution of the SDK from PyPi')
# Get a wheel distribution for the SDK from PyPI.
cmd_args.extend([
'--only-binary', ':all:', '--python-version',
language_version_tag, '--implementation',
language_implementation_tag, '--abi', abi_tag, '--platform',
platform_tag
])
# Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
expected_files = [
os.path.join(
temp_dir, '%s-%s-%s%s-%s-%s.whl' %
(package_name.replace(
'-', '_'), version, language_implementation_tag,
language_version_tag, abi_tag, platform_tag))
]
else:
logging.info(
'Downloading source distribtution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
expected_files = [
os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)),
os.path.join(temp_dir,
'%s-%s.tar.gz' % (package_name, version))
]
logging.info('Executing command: %s', cmd_args)
try:
processes.check_call(cmd_args)
except subprocess.CalledProcessError as e:
raise RuntimeError(repr(e))
for sdk_file in expected_files:
if os.path.exists(sdk_file):
return sdk_file
raise RuntimeError(
'Failed to download a distribution for the running SDK. '
'Expected either one of %s to be found in the download folder.' %
(expected_files))
def test_oserror_check_call(self):
# Configure the mock to return a response with an OK status code.
self.mock_get.side_effect = OSError("Test OSError")
with self.assertRaises(RuntimeError):
processes.check_call(["lls"])