Example #1
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')

    layer.caas_base.pod_spec_set({
        'version':
        2,
        'containers': [{
            'name':
            'minio',
            'args': ['server', '/data'],
            'imageDetails': {
                'imagePath': image_info.registry_path,
                'username': image_info.username,
                'password': image_info.password,
            },
            'ports': [{
                'name': 'minio',
                'containerPort': hookenv.config('port')
            }],
            'config': {
                'MINIO_ACCESS_KEY': hookenv.config('access-key'),
                'MINIO_SECRET_KEY': hookenv.config('secret-key'),
            },
        }],
    })

    layer.status.maintenance('creating container')
    set_flag('charm.started')
Example #2
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')

    api = endpoint_from_name('metadata-api').services()[0]
    grpc = endpoint_from_name('metadata-grpc').services()[0]

    port = hookenv.config('port')

    layer.caas_base.pod_spec_set({
        'version':
        2,
        'serviceAccount': {
            'rules': [
                {
                    'apiGroups': [''],
                    'resources': ['pods', 'pods/log'],
                    'verbs': ['create', 'get', 'list'],
                },
                {
                    'apiGroups': ['kubeflow.org'],
                    'resources': ['viewers'],
                    'verbs': ['create', 'get', 'list', 'watch', 'delete'],
                },
            ]
        },
        'containers': [{
            'name': 'metadata-ui',
            'imageDetails': {
                'imagePath': image_info.registry_path,
                'username': image_info.username,
                'password': image_info.password,
            },
            'ports': [{
                'name': 'http',
                'containerPort': port
            }],
            'config': {
                'METADATA_SERVICE_SERVICE_HOST': api['service_name'],
                'METADATA_SERVICE_SERVICE_PORT': api['hosts'][0]['port'],
                'METADATA_ENVOY_SERVICE_SERVICE_HOST': grpc['service_name'],
                'METADATA_ENVOY_SERVICE_SERVICE_PORT':
                grpc['hosts'][0]['port'],
            },
        }],
    })

    layer.status.maintenance('creating container')
    set_flag('charm.started')
Example #3
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    if not Path('/run/password').exists():
        Path('/run/password').write_text(''.join(
            choices(ascii_uppercase + digits, k=30)))

    image_info = layer.docker_resource.get_info('oci-image')
    config = dict(hookenv.config())

    layer.caas_base.pod_spec_set({
        'version':
        2,
        'containers': [{
            'name':
            'minio',
            'args': ['server', '/data'],
            'imageDetails': {
                'imagePath': image_info.registry_path,
                'username': image_info.username,
                'password': image_info.password,
            },
            'ports': [{
                'name': 'minio',
                'containerPort': config['port']
            }],
            'config': {
                'MINIO_ACCESS_KEY': config['access-key'],
                'MINIO_SECRET_KEY': Path('/run/password').read_text(),
            },
        }],
    })

    layer.status.maintenance('creating container')
    set_flag('charm.started')
Example #4
0
def start_charm():
    layer.status.maintenance("configuring container")

    image_info = layer.docker_resource.get_info("oci-image")
    namespace = os.environ["JUJU_MODEL_NAME"]

    try:
        pilot = endpoint_from_name("istio-pilot").services()[0]
    except IndexError:
        hookenv.log("Got istio-pilot endpoint without any services")
        layer.status.maintenance("Waiting for istio-pilot relation")
        return False

    pilot_service = pilot['hosts'][0]['hostname']
    pilot_port = pilot['hosts'][0]['port']
    cfg = dict(hookenv.config())

    # Talk to the K8s API to read the auto-generated root certificate secret.
    # Borrow the env vars from the root process that let the Kubernetes
    # client automatically look up connection info, since `load_incluster_config`
    # for whatever reason doesn't support loading the serviceaccount token from disk.
    os.environ.update(
        dict(
            e.split("=")
            for e in Path("/proc/1/environ").read_text().split("\x00")
            if "KUBERNETES_SERVICE" in e))

    config.load_incluster_config()
    v1 = client.CoreV1Api()
    layer.status.maintenance(
        "Waiting for configmap/istio-ca-root-cert to be created")
    try:
        config_map = v1.read_namespaced_config_map(name="istio-ca-root-cert",
                                                   namespace=namespace)
        if not config_map.data.get("root-cert.pem"):
            hookenv.log("Got empty certificate, waiting for real one")
            return False
    except client.rest.ApiException as err:
        hookenv.log(err)
        hookenv.log(err.status)
        hookenv.log(err.reason)
        hookenv.log(err.body)
        hookenv.log(err.headers)
        layer.status.blocked("istio-ca-root-cert certificate not found.")
        return False

    layer.caas_base.pod_spec_set({
        "version":
        3,
        "serviceAccount": {
            "roles": [{
                "global":
                True,
                "rules": [
                    {
                        "apiGroups": ["*"],
                        "resources": ["*"],
                        "verbs": ["*"]
                    },
                    {
                        "nonResourceURLs": ["*"],
                        "verbs": ["*"]
                    },
                ],
            }]
        },
        "containers": [{
            "name":
            "istio-proxy",
            "args": [
                "proxy",
                "router",
                "--domain",
                f"{namespace}.svc.cluster.local",
                "--proxyLogLevel=warning",
                "--proxyComponentLogLevel=misc:error",
                f"--log_output_level={cfg['log-level']}",
                "--drainDuration",
                "45s",
                "--parentShutdownDuration",
                "1m0s",
                "--connectTimeout",
                "10s",
                "--serviceCluster",
                hookenv.service_name(),
                "--proxyAdminPort",
                cfg['proxy-admin-port'],
                "--statusPort",
                str(cfg['status-port']),
                "--controlPlaneAuthPolicy",
                "NONE",
                "--discoveryAddress",
                f"{pilot_service}:{pilot_port}",
                "--trust-domain=cluster.local",
            ],
            "imageDetails": {
                "imagePath": image_info.registry_path,
                "username": image_info.username,
                "password": image_info.password,
            },
            "envConfig": {
                "JWT_POLICY": "first-party-jwt",
                "PILOT_CERT_PROVIDER": "istiod",
                "ISTIO_META_USER_SDS": "true",
                "CA_ADDR": f"{pilot_service}:{pilot_port}",
                "NODE_NAME": {
                    "field": {
                        "path": "spec.nodeName",
                        "api-version": "v1"
                    }
                },
                "POD_NAME": {
                    "field": {
                        "path": "metadata.name",
                        "api-version": "v1"
                    }
                },
                "POD_NAMESPACE": namespace,
                "INSTANCE_IP": {
                    "field": {
                        "path": "status.podIP",
                        "api-version": "v1"
                    }
                },
                "HOST_IP": {
                    "field": {
                        "path": "status.hostIP",
                        "api-version": "v1"
                    }
                },
                "SERVICE_ACCOUNT": {
                    "field": {
                        "path": "spec.serviceAccountName",
                        "api-version": "v1"
                    }
                },
                "ISTIO_META_WORKLOAD_NAME": hookenv.service_name(),
                "ISTIO_META_OWNER":
                f"kubernetes://api/apps/v1/namespaces/{namespace}/deployments/{hookenv.service_name()}",
                "ISTIO_META_MESH_ID": "cluster.local",
                "ISTIO_AUTO_MTLS_ENABLED": "true",
                "ISTIO_META_POD_NAME": {
                    "field": {
                        "path": "metadata.name",
                        "api-version": "v1"
                    }
                },
                "ISTIO_META_CONFIG_NAMESPACE": namespace,
                "ISTIO_META_ROUTER_MODE": "sni-dnat",
                "ISTIO_META_CLUSTER_ID": "Kubernetes",
            },
            "ports": [
                {
                    "name": "status-port",
                    "containerPort": cfg['status-port']
                },
                {
                    "name": "http2",
                    "containerPort": cfg['http-port']
                },
                {
                    "name": "https",
                    "containerPort": cfg['https-port']
                },
                {
                    "name": "kiali",
                    "containerPort": cfg['kiali-port']
                },
                {
                    "name": "prometheus",
                    "containerPort": cfg['prometheus-port']
                },
                {
                    "name": "grafana",
                    "containerPort": cfg['grafana-port']
                },
                {
                    "name": "tracing",
                    "containerPort": cfg['tracing-port']
                },
                {
                    "name": "tls",
                    "containerPort": cfg['tls-port']
                },
                {
                    "name": "pilot",
                    "containerPort": cfg['xds-ca-port-legacy']
                },
                {
                    "name": "citadel",
                    "containerPort": cfg['citadel-grpc-port']
                },
                {
                    "name": "dns-tls",
                    "containerPort": cfg['dns-tls-port']
                },
            ],
            "kubernetes": {
                "readinessProbe": {
                    "failureThreshold": 30,
                    "httpGet": {
                        "path": "/healthz/ready",
                        "port": cfg['status-port'],
                        "scheme": "HTTP",
                    },
                    "initialDelaySeconds": 1,
                    "periodSeconds": 2,
                    "successThreshold": 1,
                    "timeoutSeconds": 1,
                },
            },
            "volumeConfig": [
                {
                    "name":
                    "istiod-ca-cert",
                    "mountPath":
                    "/var/run/secrets/istio",
                    "files": [{
                        "path": "root-cert.pem",
                        "content": config_map.data["root-cert.pem"],
                    }],
                },
                {
                    "name": "ingressgatewaysdsudspath",
                    "mountPath": "/var/run/ingress_gateway",
                    "emptyDir": {
                        "medium": "Memory"
                    },
                },
                {
                    "name":
                    "podinfo",
                    "mountPath":
                    "/etc/istio/pod",
                    "files": [
                        {
                            "path": "annotations",
                            "content": 'sidecar.istio.io/inject="false"',
                        },
                        {
                            "path":
                            "labels",
                            "content":
                            'app="istio-ingressgateway"\nistio="ingressgateway"',
                        },
                    ],
                },
            ],
        }],
    })

    layer.status.maintenance("creating container")
    set_flag("charm.started")
Example #5
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')
    model = os.environ['JUJU_MODEL_NAME']

    profiles = endpoint_from_name('kubeflow-profiles').services()[0]
    profiles_service = profiles['service_name']

    port = hookenv.config('port')
    profile = hookenv.config('profile')

    layer.caas_base.pod_spec_set(
        {
            'version':
            2,
            'serviceAccount': {
                'global':
                True,
                'rules': [
                    {
                        'apiGroups': [''],
                        'resources': ['events', 'namespaces', 'nodes'],
                        'verbs': ['get', 'list', 'watch'],
                    },
                    {
                        'apiGroups': ['', 'app.k8s.io'],
                        'resources':
                        ['applications', 'pods', 'pods/exec', 'pods/log'],
                        'verbs': ['get', 'list', 'watch'],
                    },
                    {
                        'apiGroups': [''],
                        'resources': ['secrets'],
                        'verbs': ['get']
                    },
                ],
            },
            'containers': [{
                'name': 'kubeflow-dashboard',
                'imageDetails': {
                    'imagePath': image_info.registry_path,
                    'username': image_info.username,
                    'password': image_info.password,
                },
                'config': {
                    'USERID_HEADER': 'kubeflow-userid',
                    'USERID_PREFIX': '',
                    'PROFILES_KFAM_SERVICE_HOST':
                    f'{profiles_service}.{model}',
                    'REGISTRATION_FLOW': hookenv.config('registration-flow'),
                },
                'ports': [{
                    'name': 'ui',
                    'containerPort': port
                }],
                'kubernetes': {
                    'livenessProbe': {
                        'httpGet': {
                            'path': '/healthz',
                            'port': 8082
                        },
                        'initialDelaySeconds': 30,
                        'periodSeconds': 30,
                    }
                },
            }],
        },
        {
            'kubernetesResources': {
                'customResourceDefinitions': {
                    crd['metadata']['name']: crd['spec']
                    for crd in yaml.safe_load_all(
                        Path("files/crds.yaml").read_text())
                },
                'customResources': {
                    'profiles.kubeflow.org': [{
                        'apiVersion': 'kubeflow.org/v1beta1',
                        'kind': 'Profile',
                        'metadata': {
                            'name': profile
                        },
                        'spec': {
                            'owner': {
                                'kind': 'User',
                                'name': profile
                            }
                        },
                    }]
                },
            }
        },
    )

    layer.status.maintenance('creating container')
    set_flag('charm.started')
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')
    service_name = hookenv.service_name()

    api = endpoint_from_name('pipelines-api').services()[0]
    minio = endpoint_from_name('minio').services()[0]['hosts'][0]

    port = hookenv.config('port')

    layer.caas_base.pod_spec_set({
        'version':
        2,
        'serviceAccount': {
            'rules': [
                {
                    'apiGroups': [''],
                    'resources': ['pods', 'pods/log'],
                    'verbs': ['create', 'get', 'list'],
                },
                {
                    'apiGroups': ['kubeflow.org'],
                    'resources': ['viewers'],
                    'verbs': ['create', 'get', 'list', 'watch', 'delete'],
                },
            ]
        },
        'service': {
            'annotations': {
                'getambassador.io/config':
                yaml.dump_all([{
                    'apiVersion': 'ambassador/v0',
                    'kind': 'Mapping',
                    'name': 'pipeline-ui',
                    'prefix': '/pipeline',
                    'rewrite': '/pipeline',
                    'service': f'{service_name}:{port}',
                    'use_websocket': True,
                    'timeout_ms': 30000,
                }])
            }
        },
        'containers': [{
            'name': 'pipelines-ui',
            'imageDetails': {
                'imagePath': image_info.registry_path,
                'username': image_info.username,
                'password': image_info.password,
            },
            'config': {
                'ML_PIPELINE_SERVICE_HOST': api['service_name'],
                'ML_PIPELINE_SERVICE_PORT': api['hosts'][0]['port'],
                'MINIO_HOST': minio['hostname'],
                'MINIO_PORT': minio['port'],
                'MINIO_NAMESPACE': os.environ['JUJU_MODEL_NAME'],
                'ALLOW_CUSTOM_VISUALIZATIONS': True,
            },
            'ports': [{
                'name': 'ui',
                'containerPort': port
            }],
        }],
    })

    layer.status.maintenance('creating container')
    set_flag('charm.started')
Example #7
0
def upgrade_charm():
    """Upgrade Charm"""
    hookenv.log('Updating Tokin')
    install_tokin()
Example #8
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')

    mysql = endpoint_from_name('mysql')

    try:
        minio = endpoint_from_name('minio').mailman3()[0]
    except IndexError:
        layer.status.blocked('Waiting for minio relation.')
        return False

    if minio['ip'] is None:
        layer.status.blocked("Waiting for full minio relation.")
        return False

    hookenv.log("DEBUG")
    hookenv.log(minio)
    grpc_port = hookenv.config('grpc-port')
    http_port = hookenv.config('http-port')

    config_json = {
        'DBConfig': {
            'DriverName': 'mysql',
            'DataSourceName': mysql.database(),
            'Host': mysql.host(),
            'Port': mysql.port(),
            'User': '******',
            'Password': mysql.root_password(),
            'DBName': 'mlpipeline',
        },
        'ObjectStoreConfig': {
            'Host': minio['ip'],
            'Port': minio['port'],
            'AccessKey': minio['user'],
            'SecretAccessKey': minio['password'],
            'BucketName': hookenv.config('minio-bucket-name'),
            'Secure': False,
        },
        'InitConnectionTimeout': '5s',
        "DefaultPipelineRunnerServiceAccount": "pipeline-runner",
    }

    profiles = endpoint_from_name('kubeflow-profiles')
    if profiles and profiles.services():
        profiles = profiles.services()[0]['hosts'][0]
        config_json["PROFILES_KFAM_SERVICE_HOST"] = profiles['hostname']
        config_json["PROFILES_KFAM_SERVICE_PORT"] = profiles['port']

    viz = endpoint_from_name('pipelines-visualization')
    if viz and viz.services():
        viz = viz.services()[0]['hosts'][0]
        config_json["ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST"] = viz[
            'hostname']
        config_json["ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT"] = viz[
            'port']
    else:
        config_json["ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST"] = 'foobar'
        config_json["ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT"] = '1234'

    layer.caas_base.pod_spec_set(
        {
            'version':
            3,
            'serviceAccount': {
                'roles': [{
                    'rules': [
                        {
                            'apiGroups': ['argoproj.io'],
                            'resources': ['workflows'],
                            'verbs': [
                                'create',
                                'get',
                                'list',
                                'watch',
                                'update',
                                'patch',
                                'delete',
                            ],
                        },
                        {
                            'apiGroups': ['kubeflow.org'],
                            'resources': ['scheduledworkflows'],
                            'verbs': [
                                'create', 'get', 'list', 'update', 'patch',
                                'delete'
                            ],
                        },
                        {
                            'apiGroups': [''],
                            'resources': ['pods'],
                            'verbs': ['delete']
                        },
                    ]
                }]
            },
            'containers': [{
                'name':
                'pipelines-api',
                'imageDetails': {
                    'imagePath': image_info.registry_path,
                    'username': image_info.username,
                    'password': image_info.password,
                },
                'ports': [
                    {
                        'name': 'grpc',
                        'containerPort': grpc_port
                    },
                    {
                        'name': 'http',
                        'containerPort': http_port
                    },
                ],
                'command': [
                    'apiserver',
                    '--config=/config',
                    '--sampleconfig=/config/sample_config.json',
                    '-logtostderr=true',
                ],
                'envConfig': {
                    'POD_NAMESPACE': os.environ['JUJU_MODEL_NAME']
                },
                'volumeConfig': [
                    {
                        'name':
                        'config',
                        'mountPath':
                        '/config',
                        'files': [
                            {
                                'path': 'config.json',
                                'content': json.dumps(config_json)
                            },
                            {
                                'path':
                                'sample_config.json',
                                'content':
                                Path('files/sample_config.json').read_text(),
                            },
                        ],
                    },
                    {
                        'name':
                        'samples',
                        'mountPath':
                        '/samples',
                        'files': [{
                            'path': Path(sample).name,
                            'content': Path(sample).read_text()
                        } for sample in glob('files/*.yaml')],
                    },
                ],
            }],
        },
        k8s_resources={
            'kubernetesResources': {
                'serviceAccounts': [{
                    'name':
                    'pipeline-runner',
                    'roles': [{
                        'name':
                        'pipeline-runner',
                        'rules': [
                            {
                                'apiGroups': [''],
                                'resources': ['secrets'],
                                'verbs': ['get']
                            },
                            {
                                'apiGroups': [''],
                                'resources': ['configmaps'],
                                'verbs': ['get', 'watch', 'list'],
                            },
                            {
                                'apiGroups': [''],
                                'resources': ['persistentvolumeclaims'],
                                'verbs': ['create', 'delete', 'get'],
                            },
                            {
                                'apiGroups': ['snapshot.storage.k8s.io'],
                                'resources': ['volumesnapshots'],
                                'verbs': ['create', 'delete', 'get'],
                            },
                            {
                                'apiGroups': ['argoproj.io'],
                                'resources': ['workflows'],
                                'verbs':
                                ['get', 'list', 'watch', 'update', 'patch'],
                            },
                            {
                                'apiGroups': [''],
                                'resources':
                                ['pods', 'pods/exec', 'pods/log', 'services'],
                                'verbs': ['*'],
                            },
                            {
                                'apiGroups': ['', 'apps', 'extensions'],
                                'resources': ['deployments', 'replicasets'],
                                'verbs': ['*'],
                            },
                            {
                                'apiGroups': ['kubeflow.org'],
                                'resources': ['*'],
                                'verbs': ['*'],
                            },
                            {
                                'apiGroups': ['batch'],
                                'resources': ['jobs'],
                                'verbs': ['*']
                            },
                        ],
                    }],
                }]
            }
        },
    )

    layer.status.maintenance('creating container')
    clear_flag('mysql.changed')
    set_flag('charm.started')
Example #9
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')
    service_name = hookenv.service_name()

    crd = yaml.load(Path('files/crd-v1beta1.yaml').read_text())

    port = hookenv.config('port')

    layer.caas_base.pod_spec_set(
        {
            'version':
            2,
            'serviceAccount': {
                'global':
                True,
                'rules': [
                    {
                        'apiGroups': ['*'],
                        'resources': ['deployments', 'services'],
                        'verbs': [
                            'create', 'get', 'list', 'watch', 'update',
                            'patch', 'delete'
                        ],
                    },
                    {
                        'apiGroups': ['kubeflow.org'],
                        'resources': ['viewers'],
                        'verbs': [
                            'create', 'get', 'list', 'watch', 'update',
                            'patch', 'delete'
                        ],
                    },
                ],
            },
            'service': {
                'annotations': {
                    'getambassador.io/config':
                    yaml.dump_all([{
                        'apiVersion': 'ambassador/v0',
                        'kind': 'Mapping',
                        'name': 'pipelines-viewer',
                        'prefix': '/data',
                        'rewrite': '/data',
                        'service': f'{service_name}:{port}',
                        'use_websocket': True,
                        'timeout_ms': 30000,
                    }])
                }
            },
            'containers': [{
                'name': 'pipelines-viewer',
                'imageDetails': {
                    'imagePath': image_info.registry_path,
                    'username': image_info.username,
                    'password': image_info.password,
                },
                'config': {
                    'POD_NAMESPACE': os.environ['JUJU_MODEL_NAME']
                },
            }],
        },
        {
            'kubernetesResources': {
                'customResourceDefinitions': {
                    crd['metadata']['name']: crd['spec']
                }
            }
        },
    )

    layer.status.maintenance('creating container')
    set_flag('charm.started')
Example #10
0
def start_charm():
    if not hookenv.is_leader():
        hookenv.log("This unit is not a leader.")
        return False

    layer.status.maintenance('configuring container')

    image_info = layer.docker_resource.get_info('oci-image')

    api = endpoint_from_name('pipelines-api').services()[0]

    try:
        minio = endpoint_from_name('minio').mailman3()[0]
    except IndexError:
        layer.status.blocked('Waiting for minio relation.')
        return False

    if minio['ip'] is None:
        layer.status.blocked("Waiting for full minio relation.")
        return False

    port = hookenv.config('port')

    layer.caas_base.pod_spec_set(
        {
            'version': 2,
            'serviceAccount': {
                'rules': [
                    {
                        'apiGroups': [''],
                        'resources': ['pods', 'pods/log'],
                        'verbs': ['create', 'get', 'list'],
                    },
                    {
                        'apiGroups': ['kubeflow.org'],
                        'resources': ['viewers'],
                        'verbs': ['create', 'get', 'list', 'watch', 'delete'],
                    },
                ]
            },
            'containers': [
                {
                    'name': 'pipelines-ui',
                    'imageDetails': {
                        'imagePath': image_info.registry_path,
                        'username': image_info.username,
                        'password': image_info.password,
                    },
                    'config': {
                        'ML_PIPELINE_SERVICE_HOST': api['service_name'],
                        'ML_PIPELINE_SERVICE_PORT': api['hosts'][0]['port'],
                        'MINIO_HOST': minio['ip'],
                        'MINIO_PORT': minio['port'],
                        'MINIO_NAMESPACE': os.environ['JUJU_MODEL_NAME'],
                        'ALLOW_CUSTOM_VISUALIZATIONS': True,
                    },
                    'ports': [{'name': 'ui', 'containerPort': port}],
                }
            ],
        }
    )

    layer.status.maintenance('creating container')
    set_flag('charm.started')