def migrate(self, backupList: BackupList, manifestList: ManifestList,
**kwargs: Any) -> ManifestList:
ml = ManifestList()
for ba in backupList.backups(pluginName='secret'):
assert isinstance(ba, Backup)
metadata = V1ObjectMeta()
metadata.annotations = {}
clusterMeta = manifestList.clusterMeta()
if clusterMeta:
metadata.annotations = clusterMeta.annotations
logging.debug("Found backup {}".format(ba))
b = ba.data
fullPath = "/".join(filter(None, [b["path"], b["key"]]))
name = b["key"]
metadata.annotations[utils.namespace_path(
"secret-path")] = fullPath
metadata.name = utils.make_subdomain(name.split('/'))
sec = V1Secret(metadata=metadata)
sec.api_version = 'v1'
sec.kind = 'Secret'
# K8s requires secret values to be base64-encoded. The secret value
# is base64-encoded during backup so it can be passed as-is here.
sec.data = {utils.dnsify(name): b['value']}
manifest = Manifest(pluginName=self.plugin_name,
manifestName=utils.dnsify(fullPath))
manifest.append(sec)
ml.append(manifest)
return ml
def migrate(self, backupList: BackupList, manifestList: ManifestList,
**kwargs: Any) -> ManifestList:
node_label_tracker = NodeLabelTracker()
ml = ManifestList()
for b in backupList.backups(pluginName=self.plugin_name):
mig = MarathonMigrator(node_label_tracker=node_label_tracker,
backup=b,
backup_list=backupList,
manifest_list=manifestList)
try:
manifest = mig.migrate()
if manifest:
ml.append(manifest)
except Exception as e:
logging.warning("Cannot migrate: {}".format(e))
app_node_labels = node_label_tracker.get_apps_by_label()
if app_node_labels:
logging.info(
'Node labels used by deployments generated from Marathon apps:\n{}\n'
'Please make sure that these labels are properly set on nodes\nof the'
' target Kubernetes cluster!'.format(
json.dumps(list(app_node_labels))))
return ml
def test_docker_pull_config_secret():
pull_config_str = '{"auths":{"example.com":{"username":"******","password":"******",'\
'"email":"*****@*****.**","auth":"f00BA7"}}}'
migrated_dcos_secret = V1Secret(
kind='Secret',
api_version='v1',
metadata=V1ObjectMeta(name='nothing-depends-on-this-name'),
data={'nothing-depends-on-the-name-of-this-key': pull_config_str})
input_manifest_list = ManifestList()
input_manifest_list.append(
Manifest(pluginName="secret",
manifestName="foo.docker-c_nfig",
data=[migrated_dcos_secret]))
app = {
"id": "/foo/barify",
"container": {
"docker": {
"pullConfig": {
"secret": "pull-config"
}
}
},
"env": {
"BAR": {
"secret": "pull-config"
}
}, # See the NOTE below
"secrets": {
"pull-config": {
"source": "/foo/docker-c@nfig"
},
"unused": {
"source": "unused"
},
},
}
migrator = MarathonMigrator(object=app, manifest_list=input_manifest_list)
manifest = migrator.migrate()
# NOTE: Thit test expects that two secrets will be created:
# one for the image pull config and another for everything else.
# This might be not the optimal migration startegy.
[deployment] = [m for m in manifest if isinstance(m, V1Deployment)]
[pull_secret] = [m for m in manifest \
if isinstance(m, V1Secret) and m.type == "kubernetes.io/dockerconfigjson"]
[generic_secret] = [m for m in manifest \
if isinstance(m, V1Secret) and m.type != "kubernetes.io/dockerconfigjson"]
assert deployment.spec.template.spec.image_pull_secrets[
0].name == pull_secret.metadata.name
assert pull_secret.data[".dockerconfigjson"] == pull_config_str
assert generic_secret.data["foo.docker-c_nfig"] == pull_config_str
def test_manifest():
ml = ManifestList(path='tests/examples/simpleWithSecret')
ml.load()
assert len(ml) == 2
sec2 = ml.manifest(pluginName="secret", manifestName="test.secret2")
assert sec2 is not None
assert len(sec2) == 1
assert sec2[0].metadata.name == "test.secret2"
def test_load_manifest(tmpdir):
dir = tmpdir.mkdir("test")
list, p, b, d = create_example_list_manifest(str(dir))
list2 = ManifestList(path=str(dir))
list2.load()
# we expect different objects
assert list == list2
# but the same amount
assert len(list) == len(list2)
# and data
assert list[0][0].data == list2[0][0].data
def migrate(self, backupList: BackupList, manifestList: ManifestList,
**kwargs: T.Any) -> ManifestList:
ml = ManifestList()
for b in backupList.backups(pluginName=self.plugin_name):
mig = MetronomeMigrator(backup=b,
backup_list=backupList,
manifest_list=manifestList)
manifest = mig.migrate()
if manifest:
ml.append(manifest)
return ml
def test_load():
ml = ManifestList(path='tests/examples/simpleWithSecret')
ml.load()
secrets = [{'secret1': 'Zm9vYmFy'}, {'test.secret2': 'YmF6'}]
assert ml is not None
assert len(ml) == 2
for m in ml:
assert m is not None
assert len(m) > 0
for s in m:
assert s.kind == "Secret"
assert s.api_version == "v1"
assert s.data is not None
assert s.data in secrets
# Remove found secrets from the list to ensure they appear only once
secrets.remove(s.data)
# We found all the expected secrets
assert len(secrets) == 0
def __init__(self) -> None:
super(DCOSMigrate, self).__init__()
self.client = DCOSClient()
self.pm = PluginManager()
self.manifest_list = ManifestList()
self.backup_list = BackupList()
config = self.pm.config_options
config.extend(self.config_defaults)
self.argparse = ArgParse(
config,
prog='dcos-migrate',
usage=
'Does a backup of your DC/OS cluster and migrates everything into K8s Manifests'
)
self.phases: List[Callable[[Optional[str], bool], None]] = [
self.initPhase, self.backup, self.backup_data, self.migrate,
self.migrate_data
]
def migrate(self, backupList: BackupList, manifestList: ManifestList,
**kwargs: Any) -> ManifestList:
ml = ManifestList()
clusterBackup = backupList.backup(pluginName=self.plugin_name,
backupName='default')
if not clusterBackup:
logging.critical(
"Cluster backup not found. Cannot provide DC/OS annotations")
return ml
metadata = V1ObjectMeta(
name="dcos-{}".format(clusterBackup.data['CLUSTER_ID']))
metadata.annotations = {
utils.namespace_path("cluster-id"):
clusterBackup.data['CLUSTER_ID'],
utils.namespace_path("cluster-name"):
clusterBackup.data['CLUSTER'],
utils.namespace_path("backup-date"):
clusterBackup.data['BACKUP_DATE'],
}
cfgmap = V1ConfigMap(metadata=metadata)
# models do not set defaults -.-
cfgmap.kind = "ConfigMap"
cfgmap.api_version = "v1"
cfgmap.data = {
'MESOS_MASTER_STATE_SUMMARY_BASE64':
b64encode(
json.dumps(
clusterBackup.data['MESOS_MASTER_STATE-SUMMARY']).encode(
'ascii'))
}
manifest = Manifest(pluginName=self.plugin_name,
manifestName="dcos-cluster")
manifest.append(cfgmap)
ml.append(manifest)
return ml
def test_simple_with_secret():
ml = ManifestList(path='tests/examples/simpleWithSecret')
ml.load()
assert len(ml) == 2
with open('tests/examples/simpleWithSecret.json') as json_file:
data = json.load(json_file)
m = MarathonMigrator(object=data, manifest_list=ml)
mres = m.migrate()
assert len(m.manifest) == 3
assert mres is not None
app_id = 'group1.predictionio-server'
app_label = 'group1-predictionio-server'
assert m.manifest[0].metadata.name == app_id
assert m.manifest[1].metadata.name == app_label
assert m.manifest[
2].metadata.name == 'marathonsecret-group1.predictionio-server'
assert m.manifest[0].metadata.labels['app'] == app_label
assert m.manifest[0].spec.template.spec.containers[0].env[
0].value_from.secret_key_ref.name == "marathonsecret-group1.predictionio-server"
assert m.manifest[0].spec.template.spec.containers[0].env[
0].value_from.secret_key_ref.key == "secret1"
assert m.manifest[0].spec.template.spec.containers[0].env[
1].value_from.secret_key_ref.name == "marathonsecret-group1.predictionio-server"
assert m.manifest[0].spec.template.spec.containers[0].env[
1].value_from.secret_key_ref.key == "test.secret2"
assert m.manifest[1].kind == 'Service'
assert m.manifest[1].spec.selector['app'] == app_label
assert 'secret1' in m.manifest[2].data
assert 'test.secret2' in m.manifest[2].data
def test_secret_migrate():
with open('tests/examples/simpleSecret.json') as json_file:
data = json.load(json_file)
assert data['key'] is not None
bl = BackupList()
bl.append(Backup(pluginName='secret', backupName='foo.bar', data=data))
s = SecretPlugin()
ml = s.migrate(backupList=bl, manifestList=ManifestList())
assert len(ml) == 1
assert ml[0][0].data['foo.bar'] == 'Rk9PQkFS'
def create_example_list_manifest(dir: str) -> ManifestList:
list = ManifestList(path=str(dir))
p = "testPlugin"
b = "foobar"
metadata = V1ObjectMeta(name="secret1")
sec = V1Secret(metadata=metadata, kind="Secret", api_version="v1")
sec.data = {"secret1": "Zm9vYmFy"}
d = [sec]
list.append(Manifest(pluginName=p, manifestName=b, data=d))
list.store()
return list, p, b, d
def create_manifest_list_cluster() -> ManifestList:
clusterID = "test-1234-test-test"
ml = ManifestList()
metadata = V1ObjectMeta(name="dcos-{}".format(clusterID))
metadata.annotations = {
"migration.dcos.d2iq.com/cluster-id": clusterID,
"migration.dcos.d2iq.com/cluster-name": "testcluster",
"migration.dcos.d2iq.com/backup-date": "2021-01-25",
}
cfgmap = V1ConfigMap(metadata=metadata)
# models do not set defaults -.-
cfgmap.kind = "ConfigMap"
cfgmap.api_version = "v1"
cfgmap.data = {
"MESOS_MASTER_STATE_SUMMARY_BASE64":
b64encode(json.dumps({
"foo": "bar"
}).encode("ascii"))
}
manifest = Manifest(pluginName="cluster", manifestName="dcos-cluster")
manifest.append(cfgmap)
ml.append(manifest)
secret = Manifest(pluginName="secret", manifestName="hello-world.secret")
sec = V1Secret(metadata=V1ObjectMeta(name="hello-world.secret",
annotations=metadata.annotations))
sec.api_version = 'v1'
sec.kind = 'Secret'
sec.data = {"hello-world.secret": "Zm9vYmFy"}
secret.append(sec)
ml.append(secret)
return ml
class DCOSMigrate(object):
"""docstring for DCOSMigrate."""
phases_choices = [
"all", "backup", "backup_data", "migrate", "migrate_data"
]
config_defaults = [
Arg(name="phase",
nargs="?",
choices=phases_choices,
default="all",
positional=True,
help="phase to start with."),
Arg(name="verbose",
alternatives=["-v"],
action="count",
default=1,
help="log verbosity. Default to critical and warnings")
]
def __init__(self) -> None:
super(DCOSMigrate, self).__init__()
self.client = DCOSClient()
self.pm = PluginManager()
self.manifest_list = ManifestList()
self.backup_list = BackupList()
config = self.pm.config_options
config.extend(self.config_defaults)
self.argparse = ArgParse(
config,
prog='dcos-migrate',
usage=
'Does a backup of your DC/OS cluster and migrates everything into K8s Manifests'
)
self.phases: List[Callable[[Optional[str], bool], None]] = [
self.initPhase, self.backup, self.backup_data, self.migrate,
self.migrate_data
]
@property
def selected_phase(self) -> int:
"""returns the int(index) of the selected phase or 0"""
return self.phases_choices.index(self.pm.config['global'].get(
'phase', "all"))
def _end_process(self, message: str, exit_code: int = 0) -> int:
print("Ending DC/OS migration - {}".format(message))
return exit_code
def run(self, args: Optional[List[str]] = None) -> int:
"""main entrypoint to start the migration script. Returns exit code as int"""
self.handleArgparse(args)
self.handleGlobal()
for i, p in enumerate(self.phases):
if self.selected_phase > i:
p(None, True)
continue
p(None, False)
if self.selected_phase and self.selected_phase == i:
return self._end_process("selected phase {} reached".format(
self.phases_choices[i]))
return 0
def handleGlobal(self) -> None:
"""handle global config before starting the process"""
levels = [
logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG
]
v = self.pm.config['global'].get('verbose', 1)
level = levels[min(len(levels) - 1, v)]
logging.basicConfig(level=level, force=True)
def handleArgparse(self, args: Optional[List[str]] = None) -> None:
if args is None:
args = []
self.pm.config = self.argparse.parse_args(args)
def initPhase(self,
pluginName: Optional[str] = None,
skip: bool = False) -> None:
"""currently unused and empty method to cover all choice"""
pass
def backup(self,
pluginName: Optional[str] = None,
skip: bool = False) -> None:
if skip:
logging.info("skipping backup - trying to load from disk.")
self.backup_list.load()
return
logging.info("Calling {} Backup Batches".format(
len(self.pm.backup_batch)))
for batch in self.pm.backup_batch:
# each batch could also be executed in parallel.
# But for now just start sequential
for plugin in batch:
logging.info("Calling backup for plugin {}".format(
plugin.plugin_name))
blist = plugin.backup(client=self.client,
backupList=self.backup_list)
if blist:
self.backup_list.extend(blist)
self.backup_list.store()
def backup_data(self,
pluginName: Optional[str] = None,
skip: bool = False) -> None:
if skip:
logging.info("skipping backup data")
return
# for batch in self.pm.backup_batch:
# # each batch could also be executed in parallel.
# # But for not just start sequencial
# for plugin in batch:
# blist = plugin.backup_data(DCOSClient=self.client)
# self.backup_data_list.extend(blist)
def migrate(self,
pluginName: Optional[str] = None,
skip: bool = False) -> None:
if skip:
logging.info("skipping migrate - trying to load from disk.")
self.manifest_list.load()
return
for batch in self.pm.migrate_batch:
# each batch could also be executed in parallel.
# But for not just start sequencial
for plugin in batch:
mlist = plugin.migrate(backupList=self.backup_list,
manifestList=self.manifest_list)
if mlist:
self.manifest_list.extend(mlist)
self.manifest_list.store()
def migrate_data(self,
pluginName: Optional[str] = None,
skip: bool = False) -> None:
if skip:
logging.info("skipping migrate data")
return
# for batch in self.pm.migrate_batch:
# # each batch could also be executed in parallel.
# # But for not just start sequencial
# for plugin in batch:
# mlist = plugin.migrate(
# backupList=self.backup_list, manifestList=self.manifest_list)
# self.manifest_list.extend(mlist)
def get_plugin_names(self) -> Iterable[str]:
return self.pm.plugins.keys()