def test_GetMergedYamlData(self):
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
files = [
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml'),
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.override.yml')
]
yamlData = YamlTools.GetYamlData(files)
self.assertTrue(
'my-service' in YamlTools.GetProperties('services', yamlData))
self.assertTrue('my_repo/my.service:1.0.0' == yamlData['services']
['my-service']['image'])
self.assertTrue('new-my-service' == yamlData['services']['my-service']
['container_name'])
self.assertTrue(
len(yamlData['services']['my-service']['networks']) == 2)
self.assertTrue(
'new_network' in yamlData['services']['my-service']['networks'])
self.assertTrue('backend_network' in yamlData['services']['my-service']
['networks'])
YamlTools.DumpYamlDataToFile(
yamlData,
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'output',
'docker-compose.merged.yml'))
def MergeAndPopulateWithContainerNames(composeFiles, testComposeFile):
DockerComposeTools.MergeComposeFiles(composeFiles, testComposeFile)
yamlData = YamlTools.GetYamlData([testComposeFile])
DockerComposeTools.AddContainerNames(yamlData)
YamlTools.DumpYamlDataToFile(yamlData, testComposeFile)
containerNames = DockerComposeTools.GetContainerNames(yamlData)
return containerNames
def test_TryGetFromDictionary_success(self):
selection = {'validKey': 'value'}
value = YamlTools.TryGetFromDictionary(selection, 'validKey',
'defaultValue')
self.assertEqual(value, 'value')
value = YamlTools.TryGetFromDictionary(selection, 'invalidKey',
'defaultValue')
self.assertEqual(value, 'defaultValue')
def CreateVolume(volumeName, volumeProperties):
if volumeProperties == None:
volumeProperties = {}
driver = YamlTools.TryGetFromDictionary(volumeProperties, 'driver',
'local')
driverOptions = YamlTools.TryGetFromDictionary(volumeProperties,
'driverOptions', [])
DockerSwarmTools.CreateSwarmVolume(volumeName, driver, driverOptions)
def ExportChangelogSelection(changelogSelection):
if not (FILE_KEY in changelogSelection):
return
VersionTools.ExportVersionFromChangelogToEnvironment(
changelogSelection[FILE_KEY],
YamlTools.TryGetFromDictionary(changelogSelection, ENV_VERSION_KEY,
'VERSION'),
YamlTools.TryGetFromDictionary(changelogSelection,
ENV_VERSION_MAJOR_KEY, None),
YamlTools.TryGetFromDictionary(changelogSelection,
ENV_VERSION_MINOR_KEY, None))
def test_GetYamlData(self):
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')])
self.assertTrue(
'my-service' in YamlTools.GetProperties('services', yamlData))
self.assertTrue('my_repo/my.service:1.0.0' == yamlData['services']
['my-service']['image'])
self.assertTrue('env_value' == yamlData['services']['my-service']
['environment']['SOME_VARIABLE'])
self.assertTrue('default_value' == yamlData['services']['my-service']
['environment']['SOME_VARIABLE_2'])
def RunSelection(runSelection, selectionToRun):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(runSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
runSelection)
BuildTools.HandleTerminalCommandsSelection(runSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in runSelection:
runComposeFile = BuildTools.GetAvailableComposeFilename(
'run', selectionToRun)
composeFiles = runSelection[BuildTools.FILES_KEY]
if YamlTools.TryGetFromDictionary(runSelection,
VERIFY_CONTAINER_EXIT_CODE, False):
containerNames = BuildTools.MergeAndPopulateWithContainerNames(
composeFiles, runComposeFile)
if BuildTools.CONTAINER_NAMES_KEY in runSelection:
containerNames = runSelection[BuildTools.CONTAINER_NAMES_KEY]
else:
containerNames = []
DockerComposeTools.MergeComposeFiles(composeFiles, runComposeFile)
try:
DockerComposeTools.DockerComposeUp(
[runComposeFile],
YamlTools.TryGetFromDictionary(runSelection,
ABORT_ON_CONTAINER_EXIT_KEY,
True),
YamlTools.TryGetFromDictionary(runSelection, DETACHED_KEY,
False))
except:
BuildTools.RemoveComposeFileIfNotPreserved(runComposeFile,
runSelection)
raise
DockerImageTools.VerifyContainerExitCode(containerNames,
assertExitCodes=True)
BuildTools.HandleCopyFromContainer(runSelection)
if YamlTools.TryGetFromDictionary(runSelection,
BuildTools.REMOVE_CONTAINERS_KEY,
False):
DockerComposeTools.DockerComposeRemove([runComposeFile])
BuildTools.RemoveComposeFileIfNotPreserved(runComposeFile,
runSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def CreateNetwork(networkName, networkProperties):
if networkProperties == None:
networkProperties = {}
elif isinstance(networkProperties, bool):
networkProperties = {'encrypted': networkProperties}
encrypted = YamlTools.TryGetFromDictionary(networkProperties, 'encrypted',
False)
driver = YamlTools.TryGetFromDictionary(networkProperties, 'driver',
'overlay')
attachable = YamlTools.TryGetFromDictionary(networkProperties,
'attachable', True)
options = YamlTools.TryGetFromDictionary(networkProperties, 'options', [])
DockerSwarmTools.CreateSwarmNetwork(networkName, encrypted, driver,
attachable, options)
def GetPublishSelections(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(
arguments, BuildTools.DEFAULT_BUILD_MANAGEMENT_YAML_FILES)
publishProperty = YamlTools.GetProperties(PUBLISH_KEY, yamlData)
if BuildTools.SELECTIONS_KEY in publishProperty:
return publishProperty[BuildTools.SELECTIONS_KEY]
return {}
def __VerifyStackExecutedSuccessfully(temporaryStackFile: str, stackName: str,
version: semantic_version.Version,
noLogs: bool, logsFolder: str):
yamlData: dict = YamlTools.GetYamlData([temporaryStackFile])
sumExitCodes = 0
services = yamlData.get('services', [])
for service in services:
if not ('container_name' in yamlData['services'][service]):
raise Exception(
("Cannot check exit code for service {0} in stack {1} " +
"due to missing container name tag.").format(
service, stackName))
containerName = yamlData['services'][service]['container_name']
exitCode = DockerImageTools.GetContainerExitCode(containerName)
if not (noLogs):
__WriteLogsFromContainerToFile(containerName, service, stackName,
version, logsFolder)
sumExitCodes += exitCode
if exitCode > 0:
warnings.warn("Container '" + containerName + "' FAILED!")
else:
print(containerName + " container finished with success.")
return sumExitCodes
def GetTestSelections(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(
arguments, BuildTools.DEFAULT_BUILD_MANAGEMENT_YAML_FILES)
testProperty = YamlTools.GetProperties(TEST_KEY, yamlData)
if BuildTools.SELECTIONS_KEY in testProperty:
return testProperty[BuildTools.SELECTIONS_KEY]
return {}
def VerifyStackFile(self, stackFile: str):
yamlData = YamlTools.GetYamlData([stackFile])
valid = True
for service in yamlData.get('services', []):
if not ('image' in yamlData['services'][service]):
warnings.warn(
'Missing image in compose file: {0}'.format(stackFile))
valid = False
continue
imageName = yamlData['services'][service]['image']
if self.__verifyImageDigests:
valid &= VerificationTools.VerifyImageDigest(
stackFile, service, imageName)
if self.__verifyImages:
valid &= VerificationTools.VerifyImage(
imageName, self.__requiredImageLabels)
if self.__verifyNoConfigs:
valid &= VerificationTools.VerifyNoConfigs(yamlData, service)
if self.__verifyNoSecrets:
valid &= VerificationTools.VerifyNoSecrets(yamlData, service)
if self.__verifyNoVolumes:
valid &= VerificationTools.VerifyNoVolumes(yamlData, service)
if self.__verifyNoPorts:
valid &= VerificationTools.VerifyNoPorts(yamlData, service)
return valid
def RemoveModule(stackHandler: StackHandler, moduleFile):
yamlData: dict = YamlTools.GetYamlData([moduleFile])
modules = yamlData.get('modules', [])
for module in modules:
moduleData: dict = yamlData['modules'][module]
if 'deploy' in moduleData:
stackHandler.Remove(moduleData['deploy'])
def HandleDumpYamlData(arguments,
defaultYamlFiles=DEFAULT_SWARM_MANAGEMENT_YAML_FILES):
if not ('-dump' in arguments):
return
outputFiles = GetArgumentValues(arguments, '-dump')
for outputFile in outputFiles:
yamlData = LoadYamlDataFromFiles(arguments, defaultYamlFiles)
YamlTools.DumpYamlDataToFile(yamlData, outputFile)
def __GenerateStackFileWithContainerNames(stackFile: str, stackName: str,
version: semantic_version.Version,
stacksFolder: str):
yamlData: dict = YamlTools.GetYamlData([stackFile])
random.seed()
randomId = random.randint(0, 1000)
prefix = stackName + "_"
subfix = "_" + str(randomId)
DockerComposeTools.AddContainerNames(yamlData,
prefix=prefix,
subfix=subfix)
temporaryStackFile = os.path.join(
stacksFolder,
"docker-compose-temp.{0}.{1}.{2}.yml".format(stackName, version,
randomId))
YamlTools.DumpYamlDataToFile(yamlData, temporaryStackFile)
return temporaryStackFile
def test_g_ComposeTestWithContainerNamesNotSet(self):
print('COMPOSE TEST UNKNOWN NAME')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData([
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.yml')
])
DockerComposeTools.AddContainerNames(yamlData)
YamlTools.DumpYamlDataToFile(
yamlData,
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.temp.yml'))
DockerComposeTools.ExecuteComposeTests([
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.temp.yml')
])
print('DONE COMPOSE TEST UNKNOWN NAME')
def LoadYamlDataFromFiles(arguments, defaultYamlFiles=DEFAULT_SWARM_MANAGEMENT_YAML_FILES, \
ignoreEmptyYamlData = False):
yamlFiles = GetArgumentValues(arguments, '-file')
yamlFiles += GetArgumentValues(arguments, '-f')
if len(yamlFiles) == 0:
yamlFiles = defaultYamlFiles
yamlFiles = AssertYamlFilesExists(yamlFiles, defaultYamlFiles)
yamlData = YamlTools.GetYamlData(yamlFiles,
ignoreEmptyYamlData,
infoMsgOnError=GetInfoMsg())
return yamlData
def PromoteImageSelection(promoteSelection, selectionToPromote):
print("selection to promote: {}".format(selectionToPromote))
composeFiles = promoteSelection[BuildTools.FILES_KEY]
promoteComposeFile = BuildTools.GetAvailableComposeFilename(
'promote', selectionToPromote)
DockerComposeTools.MergeComposeFiles(composeFiles, promoteComposeFile)
sourceFeed = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.SOURCE_FEED_KEY,
None)
targetFeed = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.TARGET_FEED_KEY,
None)
user = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.USER_KEY, None)
password = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.PASSWORD_KEY, None)
logout = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.LOGOUT_KEY, False)
dryRun = YamlTools.TryGetFromDictionary(promoteSelection,
BuildTools.DRY_RUN_KEY, False)
DockerComposeTools.PromoteDockerImages(
composeFile=promoteComposeFile,
targetTags=promoteSelection[BuildTools.TARGET_TAGS_KEY],
sourceFeed=sourceFeed,
targetFeed=targetFeed,
user=user,
password=password,
logoutFromFeeds=logout,
dryRun=dryRun)
BuildTools.RemoveComposeFileIfNotPreserved(promoteComposeFile,
promoteSelection)
def VerifyModule(stackHandler: StackHandler, moduleFile):
yamlData: dict = YamlTools.GetYamlData([moduleFile])
modules = yamlData.get('modules', [])
valid = True
for module in modules:
moduleData: dict = yamlData['modules'][module]
if 'run' in moduleData:
valid &= stackHandler.Verify(moduleData['run'])
if 'deploy' in moduleData:
valid &= stackHandler.Verify(moduleData['deploy'])
return valid
def PromoteSelection(promoteSelection, selectionToPromote):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(promoteSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
promoteSelection)
BuildTools.HandleTerminalCommandsSelection(promoteSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in promoteSelection:
if YamlTools.TryGetFromDictionary(promoteSelection,
CONTAINER_ARTIFACT_KEY, True):
PromoteImageSelection(promoteSelection, selectionToPromote)
else:
print('Only image promotion is supported')
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def DeployModule(stackHandler: StackHandler, moduleFile):
yamlData: dict = YamlTools.GetYamlData([moduleFile])
modules = yamlData.get('modules', [])
for module in modules:
moduleData: dict = yamlData['modules'][module]
batchProcessesRanWithSuccess = True
if 'run' in moduleData:
batchProcessesRanWithSuccess = stackHandler.Run(moduleData['run'])
if not (batchProcessesRanWithSuccess):
baseFilename = os.path.basename(moduleFile)
warnings.warn(
"Some of the batch processes of the {0} module in module artifact {1} failed! See warnings in log."
.format(module, baseFilename))
if 'deploy' in moduleData and batchProcessesRanWithSuccess:
stackHandler.Deploy(moduleData['deploy'])
def PublishSelection(publishSelection, selectionToPublish):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(publishSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
publishSelection)
BuildTools.HandleTerminalCommandsSelection(publishSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in publishSelection:
if YamlTools.TryGetFromDictionary(publishSelection,
CONTAINER_ARTIFACT_KEY, True):
PublishContainerSelection(publishSelection, selectionToPublish)
else:
PublishArtifactSelection(publishSelection)
BuildTools.HandleCopyFromContainer(publishSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def test_h_AddDigestsToImageTags(self):
print('COMPOSE ADD DIGESTS')
DockerImageTools.PullImage('nginx')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')],
replaceEnvironmentVariablesMatches=False)
DockerComposeTools.AddDigestsToImageTags(yamlData)
for service in yamlData['services']:
if 'my.service' in yamlData['services'][service]['image']:
self.assertEqual('my_repo/my.service:1.0.0',
yamlData['services'][service]['image'])
else:
self.assertTrue(
'nginx@sha256:' in yamlData['services'][service]['image'])
self.assertTrue(yamlData['services'][service]['environment']
['SOME_VARIABLE'] == '${SOME_ENV_VARIABLE}')
print('DONE COMPOSE ADD DIGESTS')
def TestSelection(testSelection, selectionToTest):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(testSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
testSelection)
BuildTools.HandleTerminalCommandsSelection(testSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in testSelection:
testComposeFile = BuildTools.GetAvailableComposeFilename(
'test', selectionToTest)
composeFiles = testSelection[BuildTools.FILES_KEY]
containerNames = BuildTools.MergeAndPopulateWithContainerNames(
composeFiles, testComposeFile)
if BuildTools.CONTAINER_NAMES_KEY in testSelection:
containerNames = testSelection[BuildTools.CONTAINER_NAMES_KEY]
try:
DockerComposeTools.ExecuteComposeTests(
[testComposeFile],
testContainerNames=containerNames,
removeTestContainers=False,
buildCompose=True,
downCompose=False)
except:
BuildTools.RemoveComposeFileIfNotPreserved(testComposeFile,
testSelection)
raise
BuildTools.HandleCopyFromContainer(testSelection)
DockerComposeTools.DockerComposeDown([testComposeFile])
if YamlTools.TryGetFromDictionary(testSelection,
BuildTools.REMOVE_CONTAINERS_KEY,
False):
DockerComposeTools.DockerComposeRemove([testComposeFile])
BuildTools.RemoveComposeFileIfNotPreserved(testComposeFile,
testSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def GenerateComposeFileWithDigests(composeFiles, outputComposeFile):
TerminalTools.LoadDefaultEnvironmentVariablesFile()
yamlData = YamlTools.GetYamlData(composeFiles, replaceEnvironmentVariablesMatches=False)
DockerComposeTools.AddDigestsToImageTags(yamlData)
YamlTools.DumpYamlDataToFile(yamlData, outputComposeFile)
def GetChangelogSelection(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(
arguments, BuildTools.DEFAULT_BUILD_MANAGEMENT_YAML_FILES)
return YamlTools.GetProperties(CHANGELOG_KEY, yamlData)
def GetConfigs(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(arguments)
return YamlTools.GetProperties('configs', yamlData)
def GetVolumes(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(arguments)
return YamlTools.GetProperties('volumes', yamlData)
def GetStacks(arguments):
yamlData = SwarmTools.LoadYamlDataFromFiles(arguments)
return YamlTools.GetProperties('stacks', yamlData)
def test_ReplaceEnvironmentVariablesMatches(self):
os.environ['TEST_KEY'] = 'test-value'
yamlString = 'abc ${TEST_KEY} def'
yamlString = YamlTools.ReplaceEnvironmentVariablesMatches(yamlString)
self.assertFalse('${TEST_KEY}' in yamlString)
self.assertTrue('test-value' in yamlString)