def _run_image(self): self.log.info('Starting docker service from image %s', self.image) self.service = self.cli.create_service( types.TaskTemplate( container_spec=types.ContainerSpec(image=self.image, command=self.get_command(), env=self.environment, user=self.user), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit)), name='airflow-%s' % get_random_string(), labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)}) self.log.info('Service started: %s', str(self.service)) status = None # wait for the service to start the task while not self.cli.tasks(filters={'service': self.service['ID']}): continue while True: status = self.cli.tasks( filters={'service': self.service['ID']})[0]['Status']['State'] if status in ['failed', 'complete']: self.log.info('Service status before exiting: %s', status) break if self.auto_remove: self.cli.remove_service(self.service['ID']) if status == 'failed': raise AirflowException('Service failed: ' + repr(self.service))
def _run_service(self) -> None: self.log.info('Starting docker service from image %s', self.image) if not self.cli: raise Exception("The 'cli' should be initialized before!") self.service = self.cli.create_service( types.TaskTemplate( container_spec=types.ContainerSpec( image=self.image, command=self.format_command(self.command), mounts=self.mounts, env=self.environment, user=self.user, tty=self.tty, configs=self.configs, secrets=self.secrets, ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), networks=self.networks, placement=self.placement, ), name=f'airflow-{get_random_string()}', labels={'name': f'airflow__{self.dag_id}__{self.task_id}'}, mode=self.mode, ) if self.service is None: raise Exception("Service should be set here") self.log.info('Service started: %s', str(self.service)) # wait for the service to start the task while not self.cli.tasks(filters={'service': self.service['ID']}): continue if self.enable_logging: self._stream_logs_to_output() while True: if self._has_service_terminated(): self.log.info('Service status before exiting: %s', self._service_status()) break if self.service and self._service_status() != 'complete': if self.auto_remove: self.cli.remove_service(self.service['ID']) raise AirflowException('Service did not complete: ' + repr(self.service)) elif self.auto_remove: if not self.service: raise Exception("The 'service' should be initialized before!") self.cli.remove_service(self.service['ID'])
def _create_service(self): return self.cli.create_service( types.TaskTemplate( container_spec=types.ContainerSpec( image=self.image, command=self.get_command(), env=self.environment, user=self.user, tty=self.tty, configs=self._get_configs(), secrets=self._get_secrets(), ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), networks=self.networks, ), name=self._get_service_name(), labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)})
def createservice(self, service_name, host_port, container_port, image, replicas=1, command=None): # resources = types.Resources(cpu_reservation=1000000000) policy = types.RestartPolicy(condition="none") replica_mode = types.ServiceMode(mode='replicated', replicas=replicas) endpointSpec = docker.types.EndpointSpec( ports={host_port: container_port}) # service = self.client.services.create(name=service_name,image=image, command=['sleep', '120'], resources=resources, restart_policy=policy,endpoint_spec=endpointSpec,mode=replica_mode) service = self.client.services.create(name=service_name, image=image, restart_policy=policy, endpoint_spec=endpointSpec, mode=replica_mode) return service
def _run_service(self) -> None: self.log.info('Starting docker service from image %s', self.image) if not self.cli: raise Exception("The 'cli' should be initialized before!") self.service = self.cli.create_service( types.TaskTemplate( container_spec=types.ContainerSpec( image=self.image, command=self.get_command(), env=self.environment, user=self.user, tty=self.tty, ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), ), name='airflow-%s' % get_random_string(), labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)}, ) self.log.info('Service started: %s', str(self.service)) # wait for the service to start the task while not self.cli.tasks(filters={'service': self.service['ID']}): continue if self.enable_logging: self._stream_logs_to_output() while True: if self._has_service_terminated(): self.log.info('Service status before exiting: %s', self._service_status()) break if self.auto_remove: if not self.service: raise Exception("The 'service' should be initialized before!") self.cli.remove_service(self.service['ID']) if self._service_status() == 'failed': raise AirflowException('Service failed: ' + repr(self.service))
def _create_service(self, service_name, image, env=None, mounts=None, placement=None, replicas=1, mem_limit=314572800): network = self.app_config['service']['network']['name'] self.client.services.create( image, name=service_name, restart_policy=types.RestartPolicy(), env=env, mode={'Replicated': { 'Replicas': replicas }}, networks=[network], resources=types.Resources(mem_limit=mem_limit), mounts=mounts, constraints=placement)
def generate_docker_py_service_description(self, name, docker_networks): mounts = [] for mount_config in self.mounts: mounts.append( types.Mount(target=mount_config['target'], source=mount_config['source'], type=mount_config['type'], read_only=mount_config['readonly'])) configs = [] for config_config in self.configs: configs.append( types.ConfigReference(config_id=config_config['config_id'], config_name=config_config['config_name'], filename=config_config.get('filename'), uid=config_config.get('uid'), gid=config_config.get('gid'), mode=config_config.get('mode'))) secrets = [] for secret_config in self.secrets: secrets.append( types.SecretReference(secret_id=secret_config['secret_id'], secret_name=secret_config['secret_name'], filename=secret_config.get('filename'), uid=secret_config.get('uid'), gid=secret_config.get('gid'), mode=secret_config.get('mode'))) cspec = types.ContainerSpec(image=self.image, user=self.user, dns_config=types.DNSConfig( nameservers=self.dns, search=self.dns_search, options=self.dns_options), args=self.args, env=self.env, tty=self.tty, hostname=self.hostname, labels=self.container_labels, mounts=mounts, secrets=secrets, configs=configs) log_driver = types.DriverConfig(name=self.log_driver, options=self.log_driver_options) placement = types.Placement(constraints=self.constraints) restart_policy = types.RestartPolicy( condition=self.restart_policy, delay=self.restart_policy_delay, max_attempts=self.restart_policy_attempts, window=self.restart_policy_window) resources = types.Resources( cpu_limit=int(self.limit_cpu * 1000000000.0), mem_limit=self.limit_memory, cpu_reservation=int(self.reserve_cpu * 1000000000.0), mem_reservation=self.reserve_memory) update_policy = types.UpdateConfig( parallelism=self.update_parallelism, delay=self.update_delay, failure_action=self.update_failure_action, monitor=self.update_monitor, max_failure_ratio=self.update_max_failure_ratio, order=self.update_order) task_template = types.TaskTemplate(container_spec=cspec, log_driver=log_driver, restart_policy=restart_policy, placement=placement, resources=resources, force_update=self.force_update) if self.mode == 'global': self.replicas = None mode = types.ServiceMode(self.mode, replicas=self.replicas) networks = [] for network_name in self.networks: network_id = None try: network_id = list( filter(lambda n: n['name'] == network_name, docker_networks))[0]['id'] except: pass if network_id: networks.append({'Target': network_id}) else: raise Exception("no docker networks named: %s" % network_name) ports = {} for port in self.publish: ports[int(port['published_port'])] = (int(port['target_port']), port['protocol'], port['mode']) endpoint_spec = types.EndpointSpec(mode=self.endpoint_mode, ports=ports) return update_policy, task_template, networks, endpoint_spec, mode, self.labels