class StagerRealExecutableTest(sdk_test_base.WithLogCapture):
"""Tests the staging code using real executables."""
_REGISTRY = runtime_registry.Registry(
{
runtime_registry.RegistryEntry('success', {env.FLEX}):
staging._BundledCommand('success.sh', 'success.cmd'),
runtime_registry.RegistryEntry('failure', {env.FLEX}):
staging._BundledCommand('failure.sh', 'failure.cmd')
},
default=staging.NoopCommand())
_SUCCESS_OUTPUT_PATTERN = (r'-+ STDOUT -+\n'
r'out\n'
r'-+ STDERR -+\n'
r'service-yaml path: app.yaml\n'
r'app-dir path: .\n'
r'-+')
_FAILURE_PATTERN = (r'Staging command '
r'\[\S+failure.(?:sh|cmd) app.yaml . \S+\] '
r'failed with return code \[1\].\n\n'
r'-+ STDOUT -+\n'
r'out\n'
r'-+ STDERR -+\n'
r'service.yaml path: app.yaml\n'
r'app-dir path: .\n'
r'-+')
def SetUp(self):
scripts_dir = self.Resource('tests', 'unit', 'command_lib', 'app',
'testdata', 'scripts')
self.StartObjectPatch(config.Paths,
'sdk_root',
new_callable=mock.PropertyMock,
return_value=scripts_dir)
self.staging_area = tempfile.mkdtemp()
self.stager = staging.Stager(self._REGISTRY, self.staging_area)
def testStage_Success(self):
app_dir = self.stager.Stage('app.yaml', '.', 'success', env.FLEX)
self.AssertFileExistsWithContents('app.yaml contents\n',
os.path.join(app_dir, 'app.yaml'))
self.AssertLogMatches(self._SUCCESS_OUTPUT_PATTERN)
def testStage_Failure(self):
with self.assertRaisesRegex(staging.StagingCommandFailedError,
self._FAILURE_PATTERN):
self.stager.Stage('app.yaml', '.', 'failure', env.FLEX)
def testGet_MultipleEnv(self):
registry = runtime_registry.Registry({
runtime_registry.RegistryEntry('intercal', {
env.FLEX, env.STANDARD
}):
'intercal-value'
})
self.assertEqual(registry.Get('intercal', env.FLEX), 'intercal-value')
self.assertEqual(registry.Get('intercal', env.STANDARD),
'intercal-value')
def testGet_Override(self):
registry = runtime_registry.Registry(
{
runtime_registry.RegistryEntry(re.compile(r'.*'), {
env.FLEX, env.STANDARD
}):
'dummy'
},
override='my-override')
self.assertEqual(registry.Get('anything', env.FLEX), 'my-override')
self.assertEqual(registry.Get('anything', env.STANDARD), 'my-override')
def testGet_RegexpRuntime(self):
default = 'default-value'
registry = runtime_registry.Registry(
{
runtime_registry.RegistryEntry(re.compile('pattern[12]$'), {
env.FLEX
}):
'pattern-value'
},
default=default)
self.assertEqual(registry.Get('pattern1', env.FLEX), 'pattern-value')
self.assertEqual(registry.Get('pattern2', env.FLEX), 'pattern-value')
self.assertEqual(registry.Get('pattern3', env.FLEX), default)
PYTHON_TI_RUNTIME_EXPR = re.compile(r'python3\d*')
class Environment(enum.Enum):
"""Enum for different application environments.
STANDARD corresponds to App Engine Standard applications.
FLEX corresponds to any App Engine `env: flex` applications.
MANAGED_VMS corresponds to `vm: true` applications.
"""
STANDARD = 1
MANAGED_VMS = 2
FLEX = 3
def GetTiRuntimeRegistry():
"""A simple registry whose `Get()` method answers True if runtime is Ti."""
return runtime_registry.Registry(_TI_RUNTIME_REGISTRY, default=False)
STANDARD = Environment.STANDARD
FLEX = Environment.FLEX
MANAGED_VMS = Environment.MANAGED_VMS
_TI_RUNTIME_REGISTRY = {
runtime_registry.RegistryEntry(NODE_TI_RUNTIME_EXPR, {STANDARD}): True,
runtime_registry.RegistryEntry(PHP_TI_RUNTIME_EXPR, {STANDARD}): True,
runtime_registry.RegistryEntry(PYTHON_TI_RUNTIME_EXPR, {STANDARD}): True,
}
_PYTHON_GCLOUDIGNORE = '\n'.join([
gcloudignore.DEFAULT_IGNORE_FILE, '# Python pycache:', '__pycache__/',
'# Ignored by the build system', '/setup.cfg'
])
_GO_GCLOUDIGNORE = '\n'.join([
gcloudignore.DEFAULT_IGNORE_FILE, '# Binaries for programs and plugins',
'*.exe', '*.exe~', '*.dll', '*.so', '*.dylib',
'# Test binary, build with `go test -c`', '*.test',
'# Output of the go coverage tool, specifically when used with LiteIDE',
'*.out'
])
_GCLOUDIGNORE_REGISTRY = {
runtime_registry.RegistryEntry(env.NODE_TI_RUNTIME_EXPR, {env.STANDARD}):
_NODE_GCLOUDIGNORE,
runtime_registry.RegistryEntry(env.PHP_TI_RUNTIME_EXPR, {env.STANDARD}):
_PHP_GCLOUDIGNORE,
runtime_registry.RegistryEntry(env.PYTHON_TI_RUNTIME_EXPR, {env.STANDARD}):
_PYTHON_GCLOUDIGNORE,
runtime_registry.RegistryEntry(env.GO_TI_RUNTIME_EXPR, {env.STANDARD}):
_GO_GCLOUDIGNORE,
}
class SkipFilesError(core_exceptions.Error):
pass
def _GetGcloudignoreRegistry():
raise StagingCommandNotFoundError(
'The provided staging command [{}] could '
'not be found.'.format(executable))
# Path to the go-app-stager binary
_GO_APP_STAGER_DIR = os.path.join('platform', 'google_appengine')
# Path to the jar which contains the staging command
_APPENGINE_TOOLS_JAR = os.path.join('platform', 'google_appengine', 'google',
'appengine', 'tools', 'java', 'lib',
'appengine-tools-api.jar')
_STAGING_REGISTRY = {
runtime_registry.RegistryEntry(re.compile(r'(go|go1\..+)$'), {
env.FLEX, env.MANAGED_VMS
}):
_BundledCommand(os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager'),
os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager.exe'),
component='app-engine-go'),
runtime_registry.RegistryEntry(
re.compile(r'(go|go1\..+|%s)$' % env.GO_TI_RUNTIME_EXPR.pattern), {
env.STANDARD,
}):
_BundledCommand(os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager'),
os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager.exe'),
component='app-engine-go'),
runtime_registry.RegistryEntry('java-xml', {env.STANDARD}):
_BundledCommand(_APPENGINE_TOOLS_JAR,
_APPENGINE_TOOLS_JAR,
component='app-engine-java',
raise StagingCommandNotFoundError(
'The provided staging command [{}] could '
'not be found.'.format(executable))
# Path to the go-app-stager binary
_GO_APP_STAGER_DIR = os.path.join('platform', 'google_appengine')
# Path to the jar which contains the staging command
_APPENGINE_TOOLS_JAR = os.path.join('platform', 'google_appengine', 'google',
'appengine', 'tools', 'java', 'lib',
'appengine-tools-api.jar')
_STAGING_REGISTRY = {
runtime_registry.RegistryEntry(re.compile(r'(go|go1\..+)$'), {
env.FLEX, env.STANDARD, env.MANAGED_VMS
}):
_BundledCommand(os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager'),
os.path.join(_GO_APP_STAGER_DIR, 'go-app-stager.exe'),
component='app-engine-go'),
runtime_registry.RegistryEntry('java-xml', {env.STANDARD}):
_BundledCommand(_APPENGINE_TOOLS_JAR,
_APPENGINE_TOOLS_JAR,
component='app-engine-java',
mapper=_JavaStagingMapper),
}
# _STAGING_REGISTRY_BETA extends _STAGING_REGISTRY, overriding entries if the
# same key is used.
_STAGING_REGISTRY_BETA = {}
_PHP_GCLOUDIGNORE = '\n'.join([
gcloudignore.DEFAULT_IGNORE_FILE,
'# PHP Composer dependencies:',
'vendor/'
])
_PYTHON_GCLOUDIGNORE = '\n'.join([
gcloudignore.DEFAULT_IGNORE_FILE,
'# Python pycache:',
'__pycache__/'
])
_GCLOUDIGNORE_REGISTRY = {
runtime_registry.RegistryEntry(
env.NODE_TI_RUNTIME_EXPR, {env.STANDARD}): _NODE_GCLOUDIGNORE,
runtime_registry.RegistryEntry(
env.PHP_TI_RUNTIME_EXPR, {env.STANDARD}): _PHP_GCLOUDIGNORE,
runtime_registry.RegistryEntry(
env.PYTHON_TI_RUNTIME_EXPR, {env.STANDARD}): _PYTHON_GCLOUDIGNORE,
}
class SkipFilesError(core_exceptions.Error):
def __init__(self, error_message):
super(SkipFilesError, self).__init__(error_message)
def _GetGcloudignoreRegistry():
return runtime_registry.Registry(_GCLOUDIGNORE_REGISTRY, default=False)
class RuntimeRegistryTest(sdk_test_base.WithLogCapture):
_DEFAULT_REGISTRY = {
runtime_registry.RegistryEntry('intercal', {env.FLEX}):
'intercal-value',
runtime_registry.RegistryEntry('x86-asm', {env.STANDARD}):
'x86-asm-value',
}
def testGet_MatchFound(self):
registry = runtime_registry.Registry(self._DEFAULT_REGISTRY)
self.assertEqual(registry.Get('intercal', env.FLEX), 'intercal-value')
def testGet_RightRuntimeWrongEnv(self):
default = 'default-value'
registry = runtime_registry.Registry(self._DEFAULT_REGISTRY,
default=default)
self.assertEqual(
registry.Get('intercal', env.STANDARD), default,
'A matching runtime with an incorrect environment should result in the '
'default.')
def testGet_DefaultFalse(self):
registry = runtime_registry.Registry(self._DEFAULT_REGISTRY,
default=False)
self.assertFalse(
registry.Get('intercal', env.STANDARD),
'A matching runtime with an incorrect environment should result in the '
'default.')
def testGet_RegexpRuntime(self):
default = 'default-value'
registry = runtime_registry.Registry(
{
runtime_registry.RegistryEntry(re.compile('pattern[12]$'), {
env.FLEX
}):
'pattern-value'
},
default=default)
self.assertEqual(registry.Get('pattern1', env.FLEX), 'pattern-value')
self.assertEqual(registry.Get('pattern2', env.FLEX), 'pattern-value')
self.assertEqual(registry.Get('pattern3', env.FLEX), default)
def testGet_MultipleEnv(self):
registry = runtime_registry.Registry({
runtime_registry.RegistryEntry('intercal', {
env.FLEX, env.STANDARD
}):
'intercal-value'
})
self.assertEqual(registry.Get('intercal', env.FLEX), 'intercal-value')
self.assertEqual(registry.Get('intercal', env.STANDARD),
'intercal-value')
def testGet_Override(self):
registry = runtime_registry.Registry(
{
runtime_registry.RegistryEntry(re.compile(r'.*'), {
env.FLEX, env.STANDARD
}):
'dummy'
},
override='my-override')
self.assertEqual(registry.Get('anything', env.FLEX), 'my-override')
self.assertEqual(registry.Get('anything', env.STANDARD), 'my-override')
def testGet_NoMatchFound(self):
default = 'default-value'
registry = runtime_registry.Registry(self._DEFAULT_REGISTRY,
default=default)
self.assertEqual(
registry.Get('bad', env.FLEX), default,
'A non-matching runtime should always result in the default.')
def testGet_NoMatchFoundNoDefault(self):
registry = runtime_registry.Registry(self._DEFAULT_REGISTRY)
self.assertEqual(
registry.Get('bad', env.FLEX), None,
'A non-matching runtime should return None if no default is provided.'
)
class StagerRegistryTest(sdk_test_base.WithLogCapture):
"""Ensure that default- and beta staging registries works as intended."""
# Use fake command strings in order to easier compare for equality
_DEFAULT_REGISTRY = {
runtime_registry.RegistryEntry('intercal', {env.FLEX}):
'fake-intercal-command',
runtime_registry.RegistryEntry('x86-asm', {env.STANDARD}):
'fake-x86-asm-command',
}
_REGISTRY_BETA = {
runtime_registry.RegistryEntry('intercal', {env.FLEX}):
'fake-intercal-beta-command',
runtime_registry.RegistryEntry('chicken', {env.STANDARD}):
'fake-chicken-beta-command',
}
def _MockDefaultRegistries(self):
self.StartObjectPatch(staging,
'_STAGING_REGISTRY',
new=self._DEFAULT_REGISTRY)
self.StartObjectPatch(staging,
'_STAGING_REGISTRY_BETA',
new=self._REGISTRY_BETA)
def SetUp(self):
self.staging_area = '/staging-area'
def testRegistry_DefaultMappings(self):
"""Ensures the default registry works is what it was originally set to."""
self._MockDefaultRegistries()
expected = {
('intercal', env.FLEX): 'fake-intercal-command',
('x86-asm', env.STANDARD): 'fake-x86-asm-command',
('chicken', env.STANDARD): staging.NoopCommand(),
}
registry = staging.GetRegistry()
for key, value in expected.items():
self.assertEqual(registry.Get(*key), value)
def testRegistry_BetaMappings(self):
"""Ensures entries in beta- overrides entries in default-registry."""
self._MockDefaultRegistries()
expected = {
('intercal', env.FLEX): 'fake-intercal-beta-command',
('x86-asm', env.STANDARD): 'fake-x86-asm-command',
('chicken', env.STANDARD): 'fake-chicken-beta-command',
}
registry = staging.GetBetaRegistry()
for key, value in expected.items():
self.assertEqual(registry.Get(*key), value)
def testRegistry_FlexGoRegexp(self):
"""Tests that the regexp for the Go staging entry does the right thing."""
registry = staging.GetRegistry()
def Good(runtime):
command = registry.Get(runtime, env.FLEX)
self.assertFalse(isinstance(command, staging.NoopCommand))
def Bad(runtime):
command = registry.Get(runtime, env.FLEX)
self.assertTrue(isinstance(command, staging.NoopCommand))
Good('go')
Good('go1.7')
Good('go1.8')
Good('go1.8.9')
Good('go1.8a')
Good('go1.9')
Good('go1.10')
Good('go1.abc')
Bad('go1')
Bad('go2')
Bad('go2.')
Bad('go10')
Bad('go1.')
Bad('something-ends-with-go1.8')
Bad('gs://my-bucket/go1.8')
def testRegistry_StandardGoRegexp(self):
"""Tests that the regexp for the Go staging entry does the right thing."""
registry = staging.GetRegistry()
def Good(runtime):
command = registry.Get(runtime, env.STANDARD)
self.assertFalse(isinstance(command, staging.NoopCommand),
'\'%s\' should be valid' % runtime)
def Bad(runtime):
command = registry.Get(runtime, env.STANDARD)
self.assertTrue(isinstance(command, staging.NoopCommand),
'\'%s\' should be invalid' % runtime)
Good('go')
Good('go1.7')
Good('go1.8')
Good('go1.8.9')
Good('go1.8a')
Good('go1.9')
Good('go1.10')
Good('go1.abc')
Good('go110')
Good('go111')
Good('go111beta1')
Good('go111rc2')
Bad('go1')
Bad('go2')
Bad('go2.')
Bad('go10')
Bad('go1.')
Bad('something-ends-with-go1.8')
Bad('gs://my-bucket/go1.8')
class StagerMockExecTest(sdk_test_base.WithLogCapture):
"""Tests the staging code by mocking executables."""
_REGISTRY = runtime_registry.Registry(
{
runtime_registry.RegistryEntry('intercal', {env.FLEX}):
staging._BundledCommand('intercal-flex', 'intercal-flex.exe'),
runtime_registry.RegistryEntry('x86-asm', {env.STANDARD}):
staging._BundledCommand('x86-asm-standard', 'x86-asm-standard.exe',
'app-engine-x86-asm'),
runtime_registry.RegistryEntry('german', {env.STANDARD}):
staging._BundledCommand('german-standard',
'german-standard.exe',
mapper=_GermanMapper),
},
default=staging.NoopCommand())
_OUTPUT_PATTERN = (r'-+ STDOUT -+\n'
r'out\n'
r'-+ STDERR -+\n'
r'err\n'
r'-+')
_SUCCESS_MESSAGE = 'Executing staging command: [{command}]'
_ERROR_MESSAGE = (
'Staging command [{command}] failed with return code [{code}].')
def SetUp(self):
self.staging_area = '/staging-area'
self.stager = staging.Stager(self._REGISTRY, self.staging_area)
self.exec_mock = self.StartObjectPatch(execution_utils,
'Exec',
side_effect=_FakeExec())
self.sdk_root_mock = self.StartPropertyPatch(
config.Paths, 'sdk_root', return_value='sdk_root_dir')
self.mkdtemp_mock = self.StartObjectPatch(tempfile,
'mkdtemp',
autospec=True,
return_value='tmp_dir')
self.StartObjectPatch(platforms.OperatingSystem,
'Current',
return_value=platforms.OperatingSystem.LINUX)
def testStage_StagingDir(self):
"""Ensures that the staging dir is created properly."""
app_dir = self.stager.Stage('app.yaml', 'dir', 'intercal', env.FLEX)
self.assertEqual(app_dir, 'tmp_dir')
self.mkdtemp_mock.assert_called_once_with(dir='/staging-area')
def testStage_MismatchedRuntime(self):
self.assertIsNone(
self.stager.Stage('app.yaml', 'dir', 'intercal', env.STANDARD))
self.AssertOutputEquals('')
self.AssertLogNotContains('err') # Log will have debug messages
self.exec_mock.assert_not_called()
self.mkdtemp_mock.assert_not_called()
def testStage_MismatchedEnvironment(self):
self.assertIsNone(
self.stager.Stage('app.yaml', 'dir', 'intercal', env.STANDARD))
self.AssertOutputEquals('')
self.AssertLogNotContains('err') # Log will have debug messages
self.exec_mock.assert_not_called()
self.mkdtemp_mock.assert_not_called()
def testStage_NoSdkRoot(self):
self.sdk_root_mock.return_value = None
with self.assertRaises(staging.NoSdkRootError):
self.stager.Stage('app.yaml', 'dir', 'intercal', env.FLEX)
self.exec_mock.assert_not_called()
def testStage_StagingCommandFailed(self):
self.exec_mock.side_effect = _FakeExec(return_code=1)
args = [
os.path.join('sdk_root_dir', 'intercal-flex'), 'app.yaml', 'dir',
'tmp_dir'
]
command = ' '.join(args)
expected_pattern = (
re.escape(self._ERROR_MESSAGE.format(command=command, code=1)) +
'\n\n' + self._OUTPUT_PATTERN)
with self.assertRaisesRegex(staging.StagingCommandFailedError,
expected_pattern):
self.stager.Stage('app.yaml', 'dir', 'intercal', env.FLEX)
self.exec_mock.assert_called_once_with(args,
no_exit=True,
out_func=mock.ANY,
err_func=mock.ANY)
def testStage_Success(self):
args = [
os.path.join('sdk_root_dir', 'intercal-flex'), 'app.yaml', 'dir',
'tmp_dir'
]
self.stager.Stage('app.yaml', 'dir', 'intercal', env.FLEX)
self.exec_mock.assert_called_once_with(args,
no_exit=True,
out_func=mock.ANY,
err_func=mock.ANY)
command = ' '.join(args)
self.AssertLogMatches(
re.escape(self._SUCCESS_MESSAGE.format(command=command)))
self.AssertLogMatches(self._OUTPUT_PATTERN)
self.mkdtemp_mock.assert_called_once_with(dir='/staging-area')
def testStage_SuccessWindows(self):
self.StartObjectPatch(platforms.OperatingSystem,
'Current',
return_value=platforms.OperatingSystem.WINDOWS)
args = [
os.path.join('sdk_root_dir', 'intercal-flex.exe'), 'app.yaml',
'dir', 'tmp_dir'
]
self.stager.Stage('app.yaml', 'dir', 'intercal', env.FLEX)
self.exec_mock.assert_called_once_with(args,
no_exit=True,
out_func=mock.ANY,
err_func=mock.ANY)
command = ' '.join(args)
self.AssertLogMatches(
re.escape(self._SUCCESS_MESSAGE.format(command=command)))
self.AssertLogMatches(self._OUTPUT_PATTERN)
self.mkdtemp_mock.assert_called_once_with(dir='/staging-area')
def testStage_SuccessInstalledComponent(self):
ensure_installed_mock = self.StartObjectPatch(
update_manager.UpdateManager, 'EnsureInstalledAndRestart')
args = [
os.path.join('sdk_root_dir', 'x86-asm-standard'), 'app.yaml',
'dir', 'tmp_dir'
]
self.stager.Stage('app.yaml', 'dir', 'x86-asm', env.STANDARD)
self.exec_mock.assert_called_once_with(args,
no_exit=True,
out_func=mock.ANY,
err_func=mock.ANY)
command = ' '.join(args)
self.AssertLogMatches(
re.escape(self._SUCCESS_MESSAGE.format(command=command)))
self.AssertLogMatches(self._OUTPUT_PATTERN)
ensure_installed_mock.assert_called_once_with(['app-engine-x86-asm'],
msg=mock.ANY)
self.mkdtemp_mock.assert_called_once_with(dir='/staging-area')
def testStage_CustomMapper(self):
"""Test that a custom mapper function can be invoked."""
args = [
os.path.join('sdk_root_dir', 'german-standard'), '-dir', 'tmp_dir',
'-yaml', 'app.yaml', 'dir'
]
self.stager.Stage('app.yaml', 'dir', 'german', env.STANDARD)
self.exec_mock.assert_called_once_with(args,
no_exit=True,
out_func=mock.ANY,
err_func=mock.ANY)
command = ' '.join(args)
self.AssertLogMatches(
re.escape(self._SUCCESS_MESSAGE.format(command=command)))
self.AssertLogMatches(self._OUTPUT_PATTERN)
self.mkdtemp_mock.assert_called_once_with(dir='/staging-area')
