def DeployStack(composeFile,
stackName,
environmentVariablesFiles=[],
withRegistryAuth=False):
for environmentVariablesFile in environmentVariablesFiles:
TerminalTools.LoadEnvironmentVariables(environmentVariablesFile)
print("Deploying stack: " + stackName)
dockerCommand = "docker stack deploy -c " + composeFile
if withRegistryAuth:
dockerCommand += " --with-registry-auth"
dockerCommand += " " + stackName
TerminalTools.ExecuteTerminalCommands([dockerCommand], True)
def test_c_ComposeRemove(self):
print('COMPOSE REMOVE')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
DockerComposeTools.DockerComposeRemove(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')])
print('DONE COMPOSE REMOVE')
def test_GetMergedYamlData(self):
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
files = [
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml'),
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.override.yml')
]
yamlData = YamlTools.GetYamlData(files)
self.assertTrue(
'my-service' in YamlTools.GetProperties('services', yamlData))
self.assertTrue('my_repo/my.service:1.0.0' == yamlData['services']
['my-service']['image'])
self.assertTrue('new-my-service' == yamlData['services']['my-service']
['container_name'])
self.assertTrue(
len(yamlData['services']['my-service']['networks']) == 2)
self.assertTrue(
'new_network' in yamlData['services']['my-service']['networks'])
self.assertTrue('backend_network' in yamlData['services']['my-service']
['networks'])
YamlTools.DumpYamlDataToFile(
yamlData,
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'output',
'docker-compose.merged.yml'))
def GetImageInfo(imageName):
terminalCommand = "docker inspect " + imageName
info = str(
TerminalTools.ExecuteTerminalCommandAndGetOutput(
terminalCommand).decode("utf-8"))
jsonInfo = json.loads(info)[0]
return jsonInfo
def BuildDocker(buildSelection):
VersionTools.ExportVersionFromChangelogToEnvironment('CHANGELOG.md', 'VERSION')
composeFiles = [
'docker-compose.yml'
]
if buildSelection == "run":
DockerComposeTools.DockerComposeUp(composeFiles)
if buildSelection == "run":
DockerComposeTools.DockerComposeUp(composeFiles)
elif buildSelection == "test":
testContainerNames = ['my-service']
DockerComposeTools.ExecuteComposeTests(composeFiles, testContainerNames)
elif buildSelection == "publish":
BuildDocker("build")
DockerComposeTools.DockerComposePush(composeFiles)
DockerComposeTools.PublishDockerImagesWithNewTag(composeFiles, 'latest')
elif buildSelection == "help":
TerminalTools.PrintAvailableCommands(AvailableCommands)
else:
print("Please provide a valid build argument: ")
BuildDocker("help")
def BuildSelection(buildSelection, selectionToBuild):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(buildSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(buildSelection)
BuildTools.HandleTerminalCommandsSelection(buildSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in buildSelection:
composeFiles = buildSelection[BuildTools.FILES_KEY]
buildComposeFile = BuildTools.GetAvailableComposeFilename('build', selectionToBuild)
DockerComposeTools.MergeComposeFiles(composeFiles, buildComposeFile)
try:
DockerComposeTools.DockerComposeBuild([buildComposeFile])
except:
BuildTools.RemoveComposeFileIfNotPreserved(buildComposeFile, buildSelection)
raise
if BuildTools.ADDITIONAL_TAG_KEY in buildSelection:
DockerComposeTools.TagImages(buildComposeFile, buildSelection[BuildTools.ADDITIONAL_TAG_KEY])
if BuildTools.ADDITIONAL_TAGS_KEY in buildSelection:
for tag in buildSelection[BuildTools.ADDITIONAL_TAGS_KEY]:
DockerComposeTools.TagImages(buildComposeFile, tag)
if SAVE_IMAGES_KEY in buildSelection:
outputFolder = buildSelection[SAVE_IMAGES_KEY]
DockerComposeTools.SaveImages(buildComposeFile, outputFolder)
if BuildTools.COMPOSE_FILE_WITH_DIGESTS_KEY in buildSelection:
composeFileWithDigests = buildSelection[BuildTools.COMPOSE_FILE_WITH_DIGESTS_KEY]
BuildTools.GenerateComposeFileWithDigests(composeFiles, composeFileWithDigests)
BuildTools.RemoveComposeFileIfNotPreserved(buildComposeFile, buildSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def StartSwarm():
if SwarmIsInitiated():
print("Swarm is already initiated.")
return
print("Starting swarm")
dockerCommand = "docker swarm init"
TerminalTools.ExecuteTerminalCommands([dockerCommand])
def CreateSwarmVolume(volumeName, driver='local', driverOptions=[]):
print("Creating volume: {0}, with driver: {1} and driver options: {2}".
format(volumeName, driver, driverOptions))
dockerCommand = "docker volume create --driver {0}".format(driver)
for driverOption in driverOptions:
dockerCommand += " --opt {0}".format(driverOption)
dockerCommand += ' {0}'.format(volumeName)
TerminalTools.ExecuteTerminalCommands([dockerCommand])
def test_f_ComposeTest(self):
print('COMPOSE TEST')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
DockerComposeTools.ExecuteComposeTests(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')],
['my-service'])
print('DONE COMPOSE TEST')
def test_d_TagImages(self):
print('COMPOSE TAG')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
DockerComposeTools.TagImages(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml'),
'1.0.0')
print('DONE COMPOSE TAG')
def DockerLogout(server, dryRun=False):
if (dryRun):
print("Would have logged out of {0}".format(server))
else:
terminalCommand = 'docker logout {0}'.format(server)
TerminalTools.ExecuteTerminalCommands(
terminalCommands=[terminalCommand],
raiseExceptionWithErrorCode=True,
printCommand=False)
def LoadEnvironmentVariables(
arguments, defaultYamlFiles=DEFAULT_SWARM_MANAGEMENT_YAML_FILES):
yamlData = LoadYamlDataFromFiles(arguments, defaultYamlFiles, True)
environmentFiles = GetEnvironmnetVariablesFiles(arguments, yamlData)
for environmentFile in environmentFiles:
if os.path.isfile(environmentFile):
TerminalTools.LoadEnvironmentVariables(environmentFile)
elif '=' in environmentFile:
key, value = environmentFile.split('=')
os.environ[key] = value
def DockerLogin(server, userName, password, dryRun=False):
if (dryRun):
print("Would have logged in to {0} with user {1}".format(
server, userName))
else:
terminalCommand = 'docker login {0} -u {1} -p {2}'.format(
server, userName, password)
TerminalTools.ExecuteTerminalCommands(
terminalCommands=[terminalCommand],
raiseExceptionWithErrorCode=True,
printCommand=False)
def BuildDocker(buildSelection):
VersionTools.ExportVersionFromChangelogToEnvironment(
'CHANGELOG.md', 'VERSION')
srcFolder = ['src', '..']
miniSwarmManagementFile = 'swarm-management.yml'
migratorComposeFiles = ['docker-compose.migrator.yml']
nugetDockerComposeFiles = ['docker-compose.publish.nuget.yml']
generalComposeFiles = [
'docker-compose.message.handler.yml',
'docker-compose.outboxcleaner.yml',
'docker-compose.message.spammer.yml'
]
if buildSelection == 'run':
os.chdir(srcFolder[0])
DockerComposeTools.DockerComposeUp(migratorComposeFiles)
DockerComposeTools.DockerComposeUp(generalComposeFiles)
os.chdir(srcFolder[1])
elif buildSelection == 'build':
os.chdir(srcFolder[0])
DockerComposeTools.DockerComposeBuild(generalComposeFiles +
migratorComposeFiles)
os.chdir(srcFolder[1])
elif buildSelection == 'test':
os.chdir(srcFolder[0])
DockerComposeTools.ExecuteComposeTests(['docker-compose.tests.yml'],
['saferebus-tests'])
os.chdir(srcFolder[1])
elif buildSelection == 'publish':
os.chdir(srcFolder[0])
DockerComposeTools.DockerComposeBuild(nugetDockerComposeFiles)
DockerComposeTools.DockerComposeUp(nugetDockerComposeFiles, False)
os.chdir(srcFolder[1])
elif buildSelection == 'start-dev':
os.chdir(srcFolder[0] + '/ServiceDependencies')
SwarmManager.HandleManagement(
['-start', '-f', miniSwarmManagementFile])
os.chdir(srcFolder[1] + '/..')
elif buildSelection == 'stop-dev':
os.chdir(srcFolder[0] + '/ServiceDependencies')
SwarmManager.HandleManagement(['-stop', '-f', miniSwarmManagementFile])
os.chdir(srcFolder[1] + '/..')
elif buildSelection == 'help':
TerminalTools.PrintAvailableCommands(AvailableCommands)
else:
print('Please provide a valid build argument: ')
BuildDocker('help')
def DeploySwarmSelection(swarmSelection, prefix):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(swarmSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
swarmSelection)
BuildTools.HandleTerminalCommandsSelection(swarmSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
SwarmManager.HandleManagement(
[prefix] + BuildSwarmManagementFilesRow(swarmSelection) +
BuildSwarmManagementPropertiesRow(swarmSelection))
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def test_f_CreateRemoveStack(self):
print('CREATE STACK')
stack = 'test-stack-' + str(random.randint(0, 10000))
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
DockerComposeTools.DockerComposeBuild(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')])
DockerSwarmTools.DeployStack(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml'),
stack)
DockerSwarmTools.RemoveStack(stack)
print('DONE CREATE STACK')
def test_e_SaveImages(self):
print('COMPOSE SAVE')
folder = os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'output')
print(folder)
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
DockerComposeTools.SaveImages(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml'),
folder)
self.assertTrue(
os.path.isfile(os.path.join(folder, 'my.service-1.0.0.tar')))
print('DONE COMPOSE SAVE')
def test_GetYamlData(self):
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')])
self.assertTrue(
'my-service' in YamlTools.GetProperties('services', yamlData))
self.assertTrue('my_repo/my.service:1.0.0' == yamlData['services']
['my-service']['image'])
self.assertTrue('env_value' == yamlData['services']['my-service']
['environment']['SOME_VARIABLE'])
self.assertTrue('default_value' == yamlData['services']['my-service']
['environment']['SOME_VARIABLE_2'])
def SignNugetPackage(nugetPackage, certificateFilePath, certificatePassword,
timestamperUrl, packageOutputSignedFolder,
allowMissingCertificateFilePath):
verbosity = "quiet"
retries = 0
limit = 3
exception = Exception("...")
print("")
while retries != limit:
try:
if os.path.exists(certificateFilePath):
terminalCommand = 'nuget sign %s -CertificatePath %s -CertificatePassword %s -Timestamper %s -OutputDirectory %s -Verbosity %s' % (
nugetPackage, certificateFilePath, certificatePassword,
timestamperUrl, packageOutputSignedFolder, verbosity)
# print("Try Sign terminalCommand:{}".format(terminalCommand))
TerminalTools.ExecuteTerminalCommands([terminalCommand], True)
print("Signed Package({}) Completed".format(nugetPackage))
return 1
else:
if allowMissingCertificateFilePath == False:
retries = limit
raise Exception(
"certificateFilePath not found, not able to sign. Set -allowMissingCertificateFilePath true to allow it."
)
print(
"Missing certificateFilePath ({}). Copy Package({}) to Output folder({}) instead."
.format(certificateFilePath, nugetPackage,
packageOutputSignedFolder))
if (os.path.exists(packageOutputSignedFolder)):
shutil.copy(nugetPackage, packageOutputSignedFolder)
else:
os.mkdir(packageOutputSignedFolder, 0o755)
shutil.copy(nugetPackage, packageOutputSignedFolder)
touch(
os.path.join(packageOutputSignedFolder,
"_notsigned-missing-certificateFilePath.txt"))
return 0
return 1
except Exception as err:
exception = err
if retries != limit:
print(">>> Retry Sign Package({}) {} of {}".format(
nugetPackage, 1 + retries, limit))
verbosity = "normal"
retries = retries + 1
raise exception
def RunSelection(runSelection, selectionToRun):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(runSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
runSelection)
BuildTools.HandleTerminalCommandsSelection(runSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in runSelection:
runComposeFile = BuildTools.GetAvailableComposeFilename(
'run', selectionToRun)
composeFiles = runSelection[BuildTools.FILES_KEY]
if YamlTools.TryGetFromDictionary(runSelection,
VERIFY_CONTAINER_EXIT_CODE, False):
containerNames = BuildTools.MergeAndPopulateWithContainerNames(
composeFiles, runComposeFile)
if BuildTools.CONTAINER_NAMES_KEY in runSelection:
containerNames = runSelection[BuildTools.CONTAINER_NAMES_KEY]
else:
containerNames = []
DockerComposeTools.MergeComposeFiles(composeFiles, runComposeFile)
try:
DockerComposeTools.DockerComposeUp(
[runComposeFile],
YamlTools.TryGetFromDictionary(runSelection,
ABORT_ON_CONTAINER_EXIT_KEY,
True),
YamlTools.TryGetFromDictionary(runSelection, DETACHED_KEY,
False))
except:
BuildTools.RemoveComposeFileIfNotPreserved(runComposeFile,
runSelection)
raise
DockerImageTools.VerifyContainerExitCode(containerNames,
assertExitCodes=True)
BuildTools.HandleCopyFromContainer(runSelection)
if YamlTools.TryGetFromDictionary(runSelection,
BuildTools.REMOVE_CONTAINERS_KEY,
False):
DockerComposeTools.DockerComposeRemove([runComposeFile])
BuildTools.RemoveComposeFileIfNotPreserved(runComposeFile,
runSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def CreateSwarmNetwork(networkName,
encrypted=False,
driver='overlay',
attachable=True,
options=[]):
print("Creating network: " + networkName)
dockerCommand = "docker network create "
dockerCommand += "--driver {0} ".format(driver)
if attachable:
dockerCommand += "--attachable "
if encrypted:
dockerCommand += "--opt encrypted "
for option in options:
dockerCommand += "{0} ".format(option)
dockerCommand += networkName
TerminalTools.ExecuteTerminalCommands([dockerCommand])
def PromoteSelection(promoteSelection, selectionToPromote):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(promoteSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
promoteSelection)
BuildTools.HandleTerminalCommandsSelection(promoteSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in promoteSelection:
if YamlTools.TryGetFromDictionary(promoteSelection,
CONTAINER_ARTIFACT_KEY, True):
PromoteImageSelection(promoteSelection, selectionToPromote)
else:
print('Only image promotion is supported')
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def PublishSelection(publishSelection, selectionToPublish):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(publishSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
publishSelection)
BuildTools.HandleTerminalCommandsSelection(publishSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in publishSelection:
if YamlTools.TryGetFromDictionary(publishSelection,
CONTAINER_ARTIFACT_KEY, True):
PublishContainerSelection(publishSelection, selectionToPublish)
else:
PublishArtifactSelection(publishSelection)
BuildTools.HandleCopyFromContainer(publishSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def test_g_ComposeTestWithContainerNamesNotSet(self):
print('COMPOSE TEST UNKNOWN NAME')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData([
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.yml')
])
DockerComposeTools.AddContainerNames(yamlData)
YamlTools.DumpYamlDataToFile(
yamlData,
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.temp.yml'))
DockerComposeTools.ExecuteComposeTests([
os.path.join(TestTools.TEST_SAMPLE_FOLDER,
'docker-compose.test.temp.yml')
])
print('DONE COMPOSE TEST UNKNOWN NAME')
def BuildDocker(buildSelection):
srcFolder = ['src', '..']
miniSwarmManagementFile = 'swarm-management.yml'
migratorComposeFiles = [
'docker-compose.migrator.yml'
]
generalComposeFiles = [
'docker-compose.yml'
]
if buildSelection == 'run':
os.chdir(srcFolder[0])
DockerComposeTools.DockerComposeUp(migratorComposeFiles)
DockerComposeTools.DockerComposeUp(generalComposeFiles)
os.chdir(srcFolder[1])
elif buildSelection == 'build':
os.chdir(srcFolder[0])
DockerComposeTools.DockerComposeBuild(generalComposeFiles + migratorComposeFiles)
os.chdir(srcFolder[1])
elif buildSelection == 'test':
os.chdir(srcFolder[0])
DockerComposeTools.ExecuteComposeTests(
['docker-compose.tests.yml'], ['kafkaoncontainers-tests'])
os.chdir(srcFolder[1])
elif buildSelection == 'start-dev':
os.chdir(srcFolder[0] + '/ServiceDependencies')
SwarmManager.HandleManagement(['-start', '-f', miniSwarmManagementFile])
os.chdir(srcFolder[1] + '/..')
elif buildSelection == 'stop-dev':
os.chdir(srcFolder[0] + '/ServiceDependencies')
SwarmManager.HandleManagement(['-stop', '-f', miniSwarmManagementFile])
os.chdir(srcFolder[1] + '/..')
elif buildSelection == 'help':
TerminalTools.PrintAvailableCommands(AvailableCommands)
else:
print('Please provide a valid build argument: ')
BuildDocker('help')
def test_h_AddDigestsToImageTags(self):
print('COMPOSE ADD DIGESTS')
DockerImageTools.PullImage('nginx')
TerminalTools.LoadEnvironmentVariables(
os.path.join(TestTools.TEST_SAMPLE_FOLDER, '.env'))
yamlData = YamlTools.GetYamlData(
[os.path.join(TestTools.TEST_SAMPLE_FOLDER, 'docker-compose.yml')],
replaceEnvironmentVariablesMatches=False)
DockerComposeTools.AddDigestsToImageTags(yamlData)
for service in yamlData['services']:
if 'my.service' in yamlData['services'][service]['image']:
self.assertEqual('my_repo/my.service:1.0.0',
yamlData['services'][service]['image'])
else:
self.assertTrue(
'nginx@sha256:' in yamlData['services'][service]['image'])
self.assertTrue(yamlData['services'][service]['environment']
['SOME_VARIABLE'] == '${SOME_ENV_VARIABLE}')
print('DONE COMPOSE ADD DIGESTS')
def TestSelection(testSelection, selectionToTest):
cwd = BuildTools.TryChangeToDirectoryAndGetCwd(testSelection)
oldEnvironmentVariable = BuildTools.AddEnvironmentVariablesFromSelection(
testSelection)
BuildTools.HandleTerminalCommandsSelection(testSelection)
TerminalTools.LoadDefaultEnvironmentVariablesFile()
if BuildTools.FILES_KEY in testSelection:
testComposeFile = BuildTools.GetAvailableComposeFilename(
'test', selectionToTest)
composeFiles = testSelection[BuildTools.FILES_KEY]
containerNames = BuildTools.MergeAndPopulateWithContainerNames(
composeFiles, testComposeFile)
if BuildTools.CONTAINER_NAMES_KEY in testSelection:
containerNames = testSelection[BuildTools.CONTAINER_NAMES_KEY]
try:
DockerComposeTools.ExecuteComposeTests(
[testComposeFile],
testContainerNames=containerNames,
removeTestContainers=False,
buildCompose=True,
downCompose=False)
except:
BuildTools.RemoveComposeFileIfNotPreserved(testComposeFile,
testSelection)
raise
BuildTools.HandleCopyFromContainer(testSelection)
DockerComposeTools.DockerComposeDown([testComposeFile])
if YamlTools.TryGetFromDictionary(testSelection,
BuildTools.REMOVE_CONTAINERS_KEY,
False):
DockerComposeTools.DockerComposeRemove([testComposeFile])
BuildTools.RemoveComposeFileIfNotPreserved(testComposeFile,
testSelection)
BuildTools.RemoveEnvironmentVariables(oldEnvironmentVariable)
os.chdir(cwd)
def VerifySignedNugetPackage(nugetPackage):
retries = 0
limit = 3
verbosity = "quiet"
exception = Exception("...")
while retries != limit:
try:
terminalCommand = 'nuget verify -Signatures %s -Verbosity %s' % (
nugetPackage, verbosity)
TerminalTools.ExecuteTerminalCommands([terminalCommand], True)
print(
"--> Verify Signed Package({}) Succeeded".format(nugetPackage))
return
except Exception as err:
exception = err
if retries != limit:
print(">>> Retry Verify Signed Package({}) {} of {}".format(
nugetPackage, 1 + retries, limit))
verbosity = "normal"
retries = retries + 1
raise exception
def GetImageId(imageName):
terminalCommand = "docker inspect --format=\"{{.Id}}\" " + imageName
imageId = str(
TerminalTools.ExecuteTerminalCommandAndGetOutput(
terminalCommand).decode("utf-8")).replace('\n', '')
return imageId
def GetImageRepoDigest(imageName):
terminalCommand = "docker inspect --format=\"{{index .RepoDigests 0}}\" " + imageName
repoDigest = str(
TerminalTools.ExecuteTerminalCommandAndGetOutput(
terminalCommand).decode("utf-8")).replace('\n', '')
return repoDigest