def install(job): from zeroos.orchestrator.sal.Node import Node from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) # at each boot recreate the complete state in the system service = job.service node = Node.from_ays(service, get_jwt_token(job.service.aysrepo)) get_version(job) job.logger.info('mount storage pool for fuse cache') poolname = '{}_fscache'.format(service.name) node.ensure_persistance(poolname) # Set host name node.client.system('hostname %s' % service.model.data.hostname).get() node.client.bash('echo %s > /etc/hostname' % service.model.data.hostname).get() job.logger.info('configure networks') for network in service.producers.get('network', []): network.executeAction('configure', args={'node_name': service.name}) stats_collector_service = get_stats_collector(service) statsdb_service = get_statsdb(service) if stats_collector_service and statsdb_service and statsdb_service.model.data.status == 'running': stats_collector_service.executeAction('install', context=job.context) node.client.bash('modprobe ipmi_si && modprobe ipmi_devintf').get()
def apply_rules(job, gwdata=None): from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.sal.gateway.firewall import Firewall, Network from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) gwdata = {} if gwdata is None else gwdata container = Container.from_ays(job.service.parent, job.context['token'], logger=job.service.logger) portforwards = gwdata.get('portforwards', []) # lets assume the public ip is the ip of the nic which has a gateway configured publicnetwork = None privatenetworks = [] for nic in gwdata["nics"]: if nic.get("config"): if nic["config"].get("gateway", None): publicnetwork = Network(nic["name"], nic["config"]["cidr"]) else: privatenetworks.append( Network(nic["name"], nic["config"]["cidr"])) if publicnetwork and privatenetworks: firewall = Firewall(container, publicnetwork, privatenetworks, portforwards) firewall.apply_rules()
def nic_shutdown(job, message): from zeroos.orchestrator.sal.Node import Node from zeroos.orchestrator.configuration import get_jwt_token service = job.service node = Node.from_ays(service, get_jwt_token(service.aysrepo)) interface = message['name'] if interface.startswith('cont'): container_id = interface.split('-')[0].replace('cont', '') for container in node.containers.list(): if str(container.id) == container_id: container_service = service.aysrepo.serviceGet( role='container', instance=container.name) container_service.model.data.status = 'networkKilled' container_service.saveAll() return else: vms = node.client.kvm.list() for vm in vms: if interface in vm['ifctargets']: vm_service = service.aysrepo.serviceGet(role='vm', instance=vm['name']) vm_service.model.data.status = 'networkKilled' vm_service.saveAll() return job.logger.info('Failed to find vm/container interface matching %s' % interface)
async def callback(jobid, level, message, flag): if '.' not in jobid: return role, sub_id = jobid.split('.', 1) if (role not in watched_roles or watched_roles[role].get('level', level) != level or watched_roles[role].get('sub_id', sub_id) != sub_id): return service_role = watched_roles[role].get('service', role) instance = watched_roles[role].get('instance', sub_id) eof = flag & 0x6 != 0 valid_message = False matched_messages = watched_roles[role].get('message', ()) for msg in matched_messages: if msg.match(message): valid_message = True if not valid_message and not (watched_roles[role]['eof'] and eof): return srv = service.aysrepo.serviceGet(role=service_role, instance=instance, die=False) if srv: args = {'message': message, 'eof': eof, 'level': level} job.context['token'] = get_jwt_token(job.service.aysrepo) handler = watched_roles[role].get('handler', 'watchdog_handler') await srv.asyncExecuteAction(handler, context=job.context, args=args)
def processChange(job): from zeroos.orchestrator.sal.grafana.grafana import Grafana from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service args = job.model.args if args.pop('changeCategory' ) != 'dataschema' or service.model.actionsState['install'] in [ 'new', 'scheduled' ]: return container = get_container(service) container_ays = Container.from_ays(container, job.context['token'], logger=service.logger) grafana = Grafana(container_ays, service.parent.model.data.redisAddr, job.service.model.data.port, job.service.model.data.url) if 'url' in args: service.model.data.url = args.get('url', service.model.data.url) if container_ays.is_running() and grafana.is_running()[0]: grafana.stop() grafana.url = service.model.data.url grafana.start() service.saveAll()
def install(job): from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token import time service = job.service job.context['token'] = get_jwt_token(service.aysrepo) container_service = service.aysrepo.serviceGet(role='container', instance=service.model.data.container) container = Container.from_ays(container_service, job.context['token'], logger=service.logger) id = container.id client = container.node.client r = client.container.backup(id, service.model.data.url) service.model.data.type = 'container' meta = { 'name': container.name, 'node': container.node.addr, 'nics': container.nics, 'hostname': container.hostname, 'flist': container.flist, 'ports': container.ports, 'host_network': container.host_network, 'storage': container.storage, 'init_processes': container.init_processes, 'privileged': container.privileged, } service.model.data.timestamp = int(time.time()) service.model.data.meta = j.data.serializer.json.dumps(meta) service.model.data.snapshot = r.get() service.saveAll()
def delete(job): from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service client = _get_client(job, job.context['token']) client.zerotier.leave(service.model.data.nwid)
def save_config(job, vdisks=None): import yaml from zeroos.orchestrator.sal.ETCD import EtcdCluster from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service config = {"servers": [service.model.data.bind]} yamlconfig = yaml.safe_dump(config, default_flow_style=False) etcd_cluster = service.aysrepo.servicesFind(role='etcd_cluster')[0] etcd = EtcdCluster.from_ays(etcd_cluster, job.context['token']) etcd.put(key="%s:cluster:conf:tlog" % service.name, value=yamlconfig) for vdisk in vdisks: vdiskstore = vdisk.parent config = { "storageClusterID": vdiskstore.model.data.blockCluster, "tlogServerClusterID": service.name, "slaveStorageClusterID": vdiskstore.model.data.slaveCluster or "", } job.logger.debug("tlogserver %s save config for vdisk %s", service, vdisk) job.logger.debug(config) yamlconfig = yaml.safe_dump(config, default_flow_style=False) etcd.put(key="%s:vdisk:conf:storage:nbd" % vdisk.name, value=yamlconfig)
def apply_config(job, gwdata=None): import ipaddress from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.sal.gateway.dhcp import DHCP from zeroos.orchestrator.configuration import get_jwt_token service = job.service job.context['token'] = get_jwt_token(job.service.aysrepo) container = Container.from_ays(job.service.parent, job.context['token'], logger=job.service.logger) gwdata = {} if gwdata is None else gwdata nics = gwdata.get('nics', []) dhcpservers = [] for nic in nics: dhcpserver = nic.get('dhcpserver') if not dhcpserver: continue cidr = ipaddress.IPv4Interface(nic['config']['cidr']) dhcpserver['subnet'] = str(cidr.network.network_address) dhcpserver['gateway'] = str(cidr.ip) dhcpserver['interface'] = nic['name'] dhcpservers.append(dhcpserver) dhcp = DHCP(container, gwdata['domain'], dhcpservers) dhcp.stop() service.model.data.status = 'halted' service.saveAll() dhcp.apply_config() service.model.data.status = 'running' service.saveAll()
def _start_nbd(job, nbdname=None): from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) # get all path to the vdisks serve by the nbdservers medias = [] if not nbdname: nbdservers = job.service.producers.get('nbdserver', None) else: nbdservers = [ job.service.aysrepo.serviceGet(role='nbdserver', instance=nbdname) ] if not nbdservers: raise j.exceptions.RuntimeError( "Failed to start nbds, no nbds created to start") nbdserver = nbdservers[0] # build full path of the nbdserver unix socket on the host filesystem container = Container.from_ays(nbdserver.parent, job.context['token'], logger=job.service.logger) if not container.is_running(): # start container nbdserver.parent.executeAction('start', context=job.context) # make sure the nbdserver is started nbdserver.executeAction('start', context=job.context) for vdisk in job.service.model.data.vdisks: url = _nbd_url(job, container, nbdserver, vdisk) medias.append({'url': url}) return medias
def start(job): from zeroos.orchestrator.sal.StorageEngine import StorageEngine from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service if service.model.data.status == 'broken': job.logger.error( 'storage_engine "%s" is marked as broken not starting' % service.name) return service.model.data.enabled = True service.saveAll() storageEngine = StorageEngine.from_ays(service, job.context['token']) storageEngine.start() if not storageEngine.is_healthy(): service.model.data.status = 'broken' service.saveAll() raise RuntimeError('storage_engine "%s" is flagged as broken' % service.name) else: service.model.data.status = 'running' service.saveAll()
def install(job): from zeroos.orchestrator.sal.Node import Node from zeroos.orchestrator.configuration import get_jwt_token # at each boot recreate the complete state in the system service = job.service node = Node.from_ays(service, get_jwt_token(job.service.aysrepo)) job.logger.info("mount storage pool for fuse cache") poolname = "{}_fscache".format(service.name) node.ensure_persistance(poolname) # Set host name node.client.system("hostname %s" % service.model.data.hostname).get() node.client.bash("echo %s > /etc/hostname" % service.model.data.hostname).get() job.logger.info("configure networks") for network in service.producers.get('network', []): job = network.getJob('configure', args={'node_name': service.name}) j.tools.async.wrappers.sync(job.execute()) stats_collector_service = get_stats_collector(service) statsdb_service = get_statsdb(service) if stats_collector_service and statsdb_service and statsdb_service.model.data.status == 'running': j.tools.async.wrappers.sync(stats_collector_service.executeAction( 'install', context=job.context))
def create_zerodisk_container_service(job, parent, service_kind): """ first check if the vdisks container for this vm exists. if not it creates it. return the container service """ from zeroos.orchestrator.configuration import get_configuration from zeroos.orchestrator.configuration import get_jwt_token import time job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service config = get_configuration(service.aysrepo) actor = service.aysrepo.actorGet("container") args = { 'node': parent.name, 'flist': config.get( '0-disk-flist', 'https://hub.gig.tech/gig-official-apps/0-disk-master.flist'), 'hostNetworking': True, } job.logger.info("create zerodisk container from %s", args['flist']) container_name = 'vdisks_{}_{}_{}_{}'.format(service.name, parent.name, service_kind, int(time.time() * 1000000)) containerservice = actor.serviceCreate(instance=container_name, args=args) # make sure the container has the right parent, the node where this vm runs. containerservice.model.changeParent(parent) containerservice.executeAction('start', context=job.context) return containerservice
def delete(job): from zeroos.orchestrator.configuration import get_jwt_token service = job.service job.service.model.data.status = "halting" job.context['token'] = get_jwt_token(job.service.aysrepo) zerostors = service.producers.get('zerostor', []) pools = service.producers.get('storagepool', []) filesystems = service.producers.get('filesystem', []) for storageEngine in zerostors: tcps = storageEngine.producers.get('tcp', []) for tcp in tcps: tcp.executeAction('drop', context=job.context) tcp.delete() container = storageEngine.parent container.executeAction('stop', context=job.context) container.delete() for filesystem in filesystems: filesystem.executeAction('delete', context=job.context) filesystem.delete() for pool in pools: pool.executeAction('delete', context=job.context) pool.delete() delete_config(job) job.logger.info("stop cluster {}".format(service.name)) job.service.model.data.status = 'empty'
def start(job): from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service container = Container.from_ays(service, job.context['token'], logger=service.logger) container.start() if container.is_running(): service.model.data.status = "running" else: raise j.exceptions.RuntimeError("container didn't start") has_zt_nic = False for nic in service.model.data.nics: if nic.type == 'zerotier': has_zt_nic = True zerotier_nic_config(service, job.logger, container, nic) if has_zt_nic and not service.model.data.identity: service.model.data.identity = container.client.zerotier.info()['secretIdentity'] service.saveAll()
def monitor(job): from zeroos.orchestrator.sal.StorageEngine import StorageEngine from zeroos.orchestrator.configuration import get_jwt_token service = job.service if service.model.actionsState['install'] == 'ok': storageEngine = StorageEngine.from_ays(service, get_jwt_token(service.aysrepo)) running, process = storageEngine.is_running() if not running: try: job.logger.warning( "storageEngine {} not running, trying to restart".format( service.name)) service.model.dbobj.state = 'error' storageEngine.start() running, _ = storageEngine.is_running() if running: service.model.dbobj.state = 'ok' except: job.logger.error( "can't restart storageEngine {} not running".format( service.name)) service.model.dbobj.state = 'error'
def monitor(job): from zeroos.orchestrator.sal.Node import Node from zeroos.orchestrator.configuration import get_jwt_token service = job.service if service.model.actionsState['install'] == 'ok': pservice = service.parent token = get_jwt_token(job.service.aysrepo) node = Node.from_ays(pservice, token) try: pool = node.storagepools.get(service.name) if not pool.mountpoint: job.context['token'] = token install(job) devices, status = pool.get_devices_and_status() service.model.data.init('devices', len(devices)) for i, device in enumerate(devices): service.model.data.devices[i] = device service.model.data.status = status service.saveAll() except ValueError: job.logger.error("pool %s doesn't exist, cant monitor pool", service.name)
def delete(job): import random from zeroos.orchestrator.sal.ETCD import EtcdCluster from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service clusterconfig = get_cluster_config(job) node = random.choice(clusterconfig['nodes']) container = create_from_template_container(job, node) try: etcd_cluster = service.aysrepo.servicesFind(role='etcd_cluster')[0] etcd_cluster = EtcdCluster.from_ays(etcd_cluster, job.context['token']) cmd = '/bin/zeroctl delete vdisk {} --config {}'.format( service.name, etcd_cluster.dialstrings) job.logger.info(cmd) result = container.client.system(cmd, id="vdisk.delete.%s" % service.name).get() if result.state != 'SUCCESS': raise j.exceptions.RuntimeError( "Failed to run zeroctl delete {} {}".format( result.stdout, result.stderr)) delete_config(job) finally: container.stop()
def uninstall(job): from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service stats_collector_service = get_stats_collector(service) if stats_collector_service: stats_collector_service.executeAction('uninstall', context=job.context) statsdb_service = get_statsdb(service) if statsdb_service and str(statsdb_service.parent) == str(service): statsdb_service.executeAction('uninstall', context=job.context) bootstraps = service.aysrepo.servicesFind(actor='bootstrap.zero-os') if bootstraps: bootstraps[0].executeAction('delete_node', args={'node_name': service.name}) # Remove etcd_cluster if this was the last node service node_services = service.aysrepo.servicesFind(role='node') if len(node_services) > 1: return for etcd_cluster_service in service.aysrepo.servicesFind( role='etcd_cluster'): etcd_cluster_service.executeAction('delete', context=job.context) etcd_cluster_service.delete()
def processChange(job): from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service args = job.model.args category = args.pop('changeCategory') if category != "dataschema" or service.model.actionsState[ 'install'] != 'ok': return if args.get('size', None): service.executeAction('resize', context=job.context, args={'size': args['size']}) if 'timestamp' in args: timestamp = args['timestamp'] if timestamp < 0: return if str(service.model.data.status) != "halted": raise j.exceptions.RuntimeError( "Failed to rollback vdisk, vdisk must be halted to rollback") if str(service.model.data.type) not in ["boot", "db"]: raise j.exceptions.RuntimeError( "Failed to rollback vdisk, vdisk must be of type boot or db") service.model.data.timestamp = -1 service.saveAll() service.executeAction('rollback', args={'timestamp': args['timestamp']}, context=job.context)
def create_from_template_container(job, parent): """ if not it creates it. return the container service """ from zeroos.orchestrator.configuration import get_configuration from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.sal.Node import Node from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) container_name = 'vdisk_{}_{}'.format(job.service.name, parent.name) node = Node.from_ays(parent, job.context['token']) config = get_configuration(job.service.aysrepo) flist = config.get( '0-disk-flist', 'https://hub.gig.tech/gig-official-apps/0-disk-master.flist') print("Creating container for flist: %s" % flist) container = Container(name=container_name, flist=flist, host_network=True, node=node) container.start() return container
def monitor(job): from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token service = job.service if service.model.actionsState['install'] == 'ok': container = Container.from_ays(job.service, get_jwt_token(job.service.aysrepo)) running = container.is_running() if not running and service.model.data.status == 'running': try: job.logger.warning("container {} not running, trying to restart".format(service.name)) service.model.dbobj.state = 'error' container.start() if container.is_running(): service.model.dbobj.state = 'ok' except: job.logger.error("can't restart container {} not running".format(service.name)) service.model.dbobj.state = 'error' elif running and service.model.data.status == 'halted': try: job.logger.warning("container {} running, trying to stop".format(service.name)) service.model.dbobj.state = 'error' container.stop() running, _ = container.is_running() if not running: service.model.dbobj.state = 'ok' except: job.logger.error("can't stop container {} is running".format(service.name)) service.model.dbobj.state = 'error'
def start_dependent_services(job): import random from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service # creates all nbd servers for each vdisk this vm uses job.logger.info("creates vdisks container for vm {}".format(service.name)) services = [ node for node in service.aysrepo.servicesFind(role="node") if node.model.data.status != "halted" ] node = random.choice(services) if len(services) > 1 and node.name == service.parent.name: node = services.index(node) services.pop(node) node = random.choice(services) tlog_container_service = create_zerodisk_container_service( job, node, "tlog") nbd_container_service = create_zerodisk_container_service( job, service.parent, "nbd") _init_zerodisk_services(job, nbd_container_service, tlog_container_service)
def delete(job): import random from zeroos.orchestrator.sal.ETCD import EtcdCluster from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service node = random.choice(get_cluster_nodes(job)) container = create_from_template_container(job, node) try: # delete disk on storage cluster find_resp = service.aysrepo.servicesFind(role="etcd_cluster") if len(find_resp) <= 0: raise j.exceptions.RuntimeError("no etcd_cluster service found") etcd_cluster = EtcdCluster.from_ays(find_resp[0], job.context["token"]) cmd = '/bin/zeroctl delete vdisk {} --config {}'.format(service.name, etcd_cluster.dialstrings) job.logger.info("delete image {}".format(service.name)) job.logger.info(cmd) result = container.client.system(cmd, id="vdisk.delete.%s" % service.name).get() if result.state != 'SUCCESS': raise j.exceptions.RuntimeError("Failed to run zeroctl delete {} {}".format(result.stdout, result.stderr)) # remove config from etcd etcd_cluster.delete(key="%s:vdisk:conf:static" % service.name) etcd_cluster.delete(key="%s:vdisk:conf:storage:tlog" % service.name) etcd_cluster.delete(key="%s:vdisk:conf:storage:nbd" % service.name) finally: container.stop()
def init(job): from zeroos.orchestrator.configuration import get_jwt_token from zeroos.orchestrator.sal.templates import render from zeroos.orchestrator.sal.StorageCluster import StorageCluster service = job.service influxdb_actor = service.aysrepo.actorGet('influxdb') args = { 'node': service.model.data.node, 'port': service.model.data.port, 'databases': ['statistics'] } influxdb_service = influxdb_actor.serviceCreate(instance='statsdb', args=args) service.consume(influxdb_service) grafana_actor = service.aysrepo.actorGet('grafana') args = { 'node': service.model.data.node, 'influxdb': ['statsdb'] } grafana_service = grafana_actor.serviceCreate(instance='statsdb', args=args) service.consume(grafana_service) dashboard_actor = service.aysrepo.actorGet('dashboard') args = { 'grafana': 'statsdb', 'dashboard': render('dashboards/overview.json') } dashboard_actor.serviceCreate(instance='overview', args=args) # Install stats_collector on all nodes stats_collector_actor = job.service.aysrepo.actorGet('stats_collector') node_services = job.service.aysrepo.servicesFind(actor='node.zero-os') for node_service in node_services: stats_collector_service = get_stats_collector_from_node(node_service) if not stats_collector_service: args = { 'node': node_service.name, 'port': job.service.model.data.port, 'ip': job.service.parent.model.data.redisAddr, } stats_collector_service = stats_collector_actor.serviceCreate(instance=node_service.name, args=args) stats_collector_service.consume(node_service) # Create storage cluster dashboards cluster_services = job.service.aysrepo.servicesFind(actor='storage_cluster') for clusterservice in cluster_services: cluster = StorageCluster.from_ays(clusterservice, get_jwt_token(service.aysrepo)) board = cluster.dashboard args = { 'grafana': 'statsdb', 'dashboard': board } dashboard_actor.serviceCreate(instance=cluster.name, args=args) stats_collector_service.consume(clusterservice)
def updateDisks(job, client, args): if args.get('disks') is None: return from zeroos.orchestrator.sal.Container import Container from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service uuid = None if service.model.data.status == 'running': domain = get_domain(job) if domain: uuid = domain['uuid'] # Get new and old disks new_disks = _diff(args['disks'], service.model.data.disks) old_disks = _diff(service.model.data.disks, args['disks']) # Do nothing if no disk change if new_disks == [] and old_disks == []: return # Set model to new data service.model.data.disks = args['disks'] vdisk_container_service = create_zerodisk_container_service( job, service.parent, "nbd") container = Container.from_ays(vdisk_container_service, job.context['token'], logger=service.logger) # Detatching and Cleaning old disks if old_disks != []: nbdserver = service.producers.get('nbdserver', [])[0] for old_disk in old_disks: url = _nbd_url(job, container, nbdserver, old_disk['vdiskid']) if uuid: client.client.kvm.detach_disk(uuid, {'url': url}) nbdserver.executeAction('install', context=job.context) # Attaching new disks if new_disks != []: _init_zerodisk_services(job, vdisk_container_service) for disk in new_disks: diskservice = service.aysrepo.serviceGet('vdisk', disk['vdiskid']) service.consume(diskservice) service.saveAll() _start_nbd(job) nbdserver = service.producers.get('nbdserver', [])[0] for disk in new_disks: media = { 'url': _nbd_url(job, container, nbdserver, disk['vdiskid']) } if disk['maxIOps']: media['iotune'] = { 'totaliopssec': disk['maxIOps'], 'totaliopssecset': True } if uuid: client.client.kvm.attach_disk(uuid, media) service.saveAll()
def install(job): from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) job.service.model.data.status = "halted" job.service.executeAction('start', context=job.context)
def start(job): from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service service.model.data.status = 'starting' service.saveAll() service.executeAction('install', context=job.context)
def stop(job): from zeroos.orchestrator.sal.ETCD import ETCD from zeroos.orchestrator.configuration import get_jwt_token job.context['token'] = get_jwt_token(job.service.aysrepo) service = job.service etcd = ETCD.from_ays(service, job.context['token'], logger=service.logger) etcd.stop()
def monitor(job): from zeroos.orchestrator.configuration import get_jwt_token if job.service.model.data.status != 'running': return job.context['token'] = get_jwt_token(job.service.aysrepo) start(job)