def test_method_forwarding_windows(self, *unused_mocks):
# Test that the correct calls are being forwarded to the subprocess module
# and that the shell=True flag is added when we are on Windows.
processes.force_shell = True
processes.call(['subprocess', 'call'], shell=False, other_arg=True)
processes.subprocess.call.assert_called_once_with(
['subprocess', 'call'],
shell=True,
other_arg=True)
processes.check_call(
['subprocess', 'check_call'],
shell=False,
other_arg=True)
processes.subprocess.check_call.assert_called_once_with(
['subprocess', 'check_call'],
shell=True,
other_arg=True)
processes.check_output(
['subprocess', 'check_output'],
shell=False,
other_arg=True)
processes.subprocess.check_output.assert_called_once_with(
['subprocess', 'check_output'],
shell=True,
other_arg=True)
processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True)
processes.subprocess.Popen.assert_called_once_with(
['subprocess', 'Popen'],
shell=True,
other_arg=True)
def test_method_forwarding_windows(self, *unused_mocks):
# Test that the correct calls are being forwarded to the subprocess module
# and that the shell=True flag is added when we are on Windows.
processes.force_shell = True
processes.call(['subprocess', 'call'], shell=False, other_arg=True)
processes.subprocess.call.assert_called_once_with(['subprocess', 'call'],
shell=True,
other_arg=True)
processes.check_call(['subprocess', 'check_call'],
shell=False,
other_arg=True)
processes.subprocess.check_call.assert_called_once_with(
['subprocess', 'check_call'], shell=True, other_arg=True)
processes.check_output(['subprocess', 'check_output'],
shell=False,
other_arg=True)
processes.subprocess.check_output.assert_called_once_with(
['subprocess', 'check_output'], shell=True, other_arg=True)
processes.Popen(['subprocess', 'Popen'], shell=False, other_arg=True)
processes.subprocess.Popen.assert_called_once_with(['subprocess', 'Popen'],
shell=True,
other_arg=True)
def _build_setup_package(setup_file, # type: str
temp_dir, # type: str
build_setup_args=None # type: Optional[List[str]]
):
# type: (...) -> str
saved_current_directory = os.getcwd()
try:
os.chdir(os.path.dirname(setup_file))
if build_setup_args is None:
build_setup_args = [
Stager._get_python_executable(),
os.path.basename(setup_file),
'sdist',
'--dist-dir',
temp_dir
]
_LOGGER.info('Executing command: %s', build_setup_args)
processes.check_output(build_setup_args)
output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz'))
if not output_files:
raise RuntimeError(
'File %s not found.' % os.path.join(temp_dir, '*.tar.gz'))
return output_files[0]
finally:
os.chdir(saved_current_directory)
def _populate_requirements_cache(requirements_file, cache_dir):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will download package dependencies.
# The apache-beam dependency is excluded from requirements cache population
# because we stage the SDK separately.
with tempfile.TemporaryDirectory() as temp_directory:
tmp_requirements_filepath = Stager.remove_dependency_from_requirements(
requirements_file=requirements_file,
dependency_to_remove='apache-beam',
temp_directory_path=temp_directory)
cmd_args = [
Stager._get_python_executable(),
'-m',
'pip',
'download',
'--dest',
cache_dir,
'-r',
tmp_requirements_filepath,
'--exists-action',
'i',
# Download from PyPI source distributions.
'--no-binary',
':all:'
]
_LOGGER.info('Executing command: %s', cmd_args)
processes.check_output(cmd_args, stderr=processes.STDOUT)
def test_oserror_check_output_message(self):
self.mock_get.side_effect = OSError()
cmd = ["lls"]
try:
processes.check_output(cmd)
except RuntimeError as error:
self.assertIn('Executable {} not found'.format(str(cmd)),\
error.args[0])
def test_oserror_check_output_message(self):
self.mock_get.side_effect = OSError()
cmd = ["lls"]
try:
processes.check_output(cmd)
except RuntimeError as error:
self.assertIn('Executable {} not found'.format(str(cmd)),\
error.args[0])
def _download_pypi_sdk_package(temp_dir,
fetch_binary=False,
language_version_tag='27',
language_implementation_tag='cp',
abi_tag='cp27mu',
platform_tag='manylinux1_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
version = pkg_resources.get_distribution(package_name).version
except pkg_resources.DistributionNotFound:
raise RuntimeError('Please set --sdk_location command-line option '
'or install a valid {} distribution.'
.format(package_name))
cmd_args = [
Stager._get_python_executable(), '-m', 'pip', 'download', '--dest',
temp_dir,
'%s==%s' % (package_name, version), '--no-deps'
]
if fetch_binary:
logging.info('Downloading binary distribtution of the SDK from PyPi')
# Get a wheel distribution for the SDK from PyPI.
cmd_args.extend([
'--only-binary', ':all:', '--python-version', language_version_tag,
'--implementation', language_implementation_tag, '--abi', abi_tag,
'--platform', platform_tag
])
# Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
expected_files = [
os.path.join(
temp_dir, '%s-%s-%s%s-%s-%s.whl' % (package_name.replace(
'-', '_'), version, language_implementation_tag,
language_version_tag, abi_tag,
platform_tag))
]
else:
logging.info('Downloading source distribtution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
expected_files = [
os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)),
os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version))
]
logging.info('Executing command: %s', cmd_args)
try:
processes.check_output(cmd_args)
except subprocess.CalledProcessError as e:
raise RuntimeError(repr(e))
for sdk_file in expected_files:
if os.path.exists(sdk_file):
return sdk_file
raise RuntimeError(
'Failed to download a distribution for the running SDK. '
'Expected either one of %s to be found in the download folder.' %
(expected_files))
def _get_default_gcp_region(self):
"""Get a default value for Google Cloud region according to
https://cloud.google.com/compute/docs/gcloud-compute/#default-properties.
If no other default can be found, returns 'us-central1'.
"""
environment_region = os.environ.get('CLOUDSDK_COMPUTE_REGION')
if environment_region:
_LOGGER.info(
'Using default GCP region %s from $CLOUDSDK_COMPUTE_REGION',
environment_region)
return environment_region
try:
cmd = ['gcloud', 'config', 'get-value', 'compute/region']
# Use subprocess.DEVNULL in Python 3.3+.
if hasattr(subprocess, 'DEVNULL'):
DEVNULL = subprocess.DEVNULL
else:
DEVNULL = open(os.devnull, 'ab')
raw_output = processes.check_output(cmd, stderr=DEVNULL)
formatted_output = raw_output.decode('utf-8').strip()
if formatted_output:
_LOGGER.info('Using default GCP region %s from `%s`',
formatted_output, ' '.join(cmd))
return formatted_output
except RuntimeError:
pass
_LOGGER.warning(
'--region not set; will default to us-central1. Future releases of '
'Beam will require the user to set --region explicitly, or else have a '
'default set via the gcloud tool. '
'https://cloud.google.com/compute/docs/regions-zones')
return 'us-central1'
def _get_default_gcp_region(self):
"""Get a default value for Google Cloud region according to
https://cloud.google.com/compute/docs/gcloud-compute/#default-properties.
If no other default can be found, returns 'us-central1'.
"""
environment_region = os.environ.get('CLOUDSDK_COMPUTE_REGION')
if environment_region:
logging.info(
'Using default GCP region %s from $CLOUDSDK_COMPUTE_REGION',
environment_region)
return environment_region
try:
cmd = ['gcloud', 'config', 'get-value', 'compute/region']
output = processes.check_output(cmd).decode('utf-8').strip()
if output:
logging.info('Using default GCP region %s from `%s`', output,
' '.join(cmd))
return output
except RuntimeError:
pass
logging.warning(
'--region not set; will default to us-central1. Future releases of '
'Beam will require the user to set --region explicitly, or else have a '
'default set via the gcloud tool. '
'https://cloud.google.com/compute/docs/regions-zones')
return 'us-central1'
def _populate_requirements_cache(
requirements_file, cache_dir, populate_cache_with_sdists=False):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will download package dependencies.
# The apache-beam dependency is excluded from requirements cache population
# because we stage the SDK separately.
with tempfile.TemporaryDirectory() as temp_directory:
tmp_requirements_filepath = Stager._remove_dependency_from_requirements(
requirements_file=requirements_file,
dependency_to_remove='apache-beam',
temp_directory_path=temp_directory)
cmd_args = [
Stager._get_python_executable(),
'-m',
'pip',
'download',
'--dest',
cache_dir,
'-r',
tmp_requirements_filepath,
'--exists-action',
'i',
'--no-deps'
]
if populate_cache_with_sdists:
cmd_args.extend(['--no-binary', ':all:'])
else:
language_implementation_tag = 'cp'
abi_suffix = 'm' if sys.version_info < (3, 8) else ''
abi_tag = 'cp%d%d%s' % (
sys.version_info[0], sys.version_info[1], abi_suffix)
platform_tag = Stager._get_platform_for_default_sdk_container()
cmd_args.extend([
'--implementation',
language_implementation_tag,
'--abi',
abi_tag,
'--platform',
platform_tag
])
_LOGGER.info('Executing command: %s', cmd_args)
processes.check_output(cmd_args, stderr=processes.STDOUT)
def _build_setup_package(setup_file, temp_dir, build_setup_args=None):
saved_current_directory = os.getcwd()
try:
os.chdir(os.path.dirname(setup_file))
if build_setup_args is None:
build_setup_args = [
Stager._get_python_executable(),
os.path.basename(setup_file), 'sdist', '--dist-dir', temp_dir
]
logging.info('Executing command: %s', build_setup_args)
processes.check_output(build_setup_args)
output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz'))
if not output_files:
raise RuntimeError(
'File %s not found.' % os.path.join(temp_dir, '*.tar.gz'))
return output_files[0]
finally:
os.chdir(saved_current_directory)
def _populate_requirements_cache(requirements_file, cache_dir):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will not download package dependencies.
cmd_args = [
Stager._get_python_executable(),
'-m',
'pip',
'download',
'--dest',
cache_dir,
'-r',
requirements_file,
'--exists-action',
'i',
# Download from PyPI source distributions.
'--no-binary',
':all:'
]
logging.info('Executing command: %s', cmd_args)
processes.check_output(cmd_args, stderr=processes.STDOUT)
def _populate_requirements_cache(requirements_file, cache_dir):
# The 'pip download' command will not download again if it finds the
# tarball with the proper version already present.
# It will get the packages downloaded in the order they are presented in
# the requirements file and will not download package dependencies.
cmd_args = [
Stager._get_python_executable(),
'-m',
'pip',
'download',
'--dest',
cache_dir,
'-r',
requirements_file,
'--exists-action',
'i',
# Download from PyPI source distributions.
'--no-binary',
':all:'
]
logging.info('Executing command: %s', cmd_args)
processes.check_output(cmd_args)
def test_check_output_pip_install_non_existing_package(self):
returncode = 1
package = "non-exsisting-package"
cmd = ['python', '-m', 'pip', 'download', '--dest', '/var',\
'{}'.format(package),\
'--no-deps', '--no-binary', ':all:']
output = "Collecting {}".format(package)
self.mock_get.side_effect = subprocess.CalledProcessError(returncode,\
cmd, output=output)
try:
output = processes.check_output(cmd)
self.fail("The test failed due to that\
no error was raised when calling process.check_call")
except RuntimeError as error:
self.assertIn("Output from execution of subprocess: {}".format(output),\
error.args[0])
self.assertIn("Pip install failed for package: {}".format(package),\
error.args[0])
def test_check_output_pip_install_non_existing_package(self):
returncode = 1
package = "non-exsisting-package"
cmd = ['python', '-m', 'pip', 'download', '--dest', '/var',\
'{}'.format(package),\
'--no-deps', '--no-binary', ':all:']
output = "Collecting {}".format(package)
self.mock_get.side_effect = subprocess.CalledProcessError(returncode,\
cmd, output=output)
try:
output = processes.check_output(cmd)
self.fail("The test failed due to that\
no error was raised when calling process.check_call")
except RuntimeError as error:
self.assertIn("Output from execution of subprocess: {}".format(output),\
error.args[0])
self.assertIn("Pip install failed for package: {}".format(package),\
error.args[0])
def _download_pypi_sdk_package(temp_dir,
fetch_binary=False,
language_version_tag='27',
language_implementation_tag='cp',
abi_tag='cp27mu',
platform_tag='manylinux1_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
version = pkg_resources.get_distribution(package_name).version
except pkg_resources.DistributionNotFound:
raise RuntimeError(
'Please set --sdk_location command-line option '
'or install a valid {} distribution.'.format(package_name))
cmd_args = [
Stager._get_python_executable(), '-m', 'pip', 'download', '--dest',
temp_dir,
'%s==%s' % (package_name, version), '--no-deps'
]
if fetch_binary:
logging.info(
'Downloading binary distribtution of the SDK from PyPi')
# Get a wheel distribution for the SDK from PyPI.
cmd_args.extend([
'--only-binary', ':all:', '--python-version',
language_version_tag, '--implementation',
language_implementation_tag, '--abi', abi_tag, '--platform',
platform_tag
])
# Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
expected_files = [
os.path.join(
temp_dir, '%s-%s-%s%s-%s-%s.whl' %
(package_name.replace(
'-', '_'), version, language_implementation_tag,
language_version_tag, abi_tag, platform_tag))
]
else:
logging.info(
'Downloading source distribtution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
expected_files = [
os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)),
os.path.join(temp_dir,
'%s-%s.tar.gz' % (package_name, version))
]
logging.info('Executing command: %s', cmd_args)
try:
processes.check_output(cmd_args)
except processes.CalledProcessError as e:
raise RuntimeError(repr(e))
for sdk_file in expected_files:
if os.path.exists(sdk_file):
return sdk_file
raise RuntimeError(
'Failed to download a distribution for the running SDK. '
'Expected either one of %s to be found in the download folder.' %
(expected_files))