def scale_service_bk(self, target_service_name, replicas, arguments):
new_mode = {'replicated': {'Replicas': replicas}} #global
services = APIClient.services()
target_service = None
for service in services:
if service["Spec"]["Name"] == target_service_name:
target_service = service
break
if not target_service is None:
task_template = types.TaskTemplate(service["Spec"]["TaskTemplate"])
old_constraints = service['Spec']['TaskTemplate']['Placement'][
'Constraints']
placement_constraints = Placement(old_constraints)
container_image = task_template['ContainerSpec']['ContainerSpec'][
'Image']
container_command = task_template['ContainerSpec'][
'ContainerSpec']['Command']
container_spec = ContainerSpec(image=container_image,
command=container_command,
args=arguments)
# Issue between Docker engine API and docker Python API !!!
adapted_template = TaskTemplate(container_spec,
placement=placement_constraints)
networks = target_service["Spec"]['Networks']
APIClient.update_service(
target_service_name,
version=target_service["Version"]["Index"],
task_template=adapted_template,
mode=new_mode,
networks=networks,
name=target_service_name)
else:
print "can't scale service :" + str(target_service_name)
def _run_image(self):
self.log.info('Starting docker service from image %s', self.image)
self.service = self.cli.create_service(
types.TaskTemplate(
container_spec=types.ContainerSpec(image=self.image,
command=self.get_command(),
env=self.environment,
user=self.user),
restart_policy=types.RestartPolicy(condition='none'),
resources=types.Resources(mem_limit=self.mem_limit)),
name='airflow-%s' % get_random_string(),
labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)})
self.log.info('Service started: %s', str(self.service))
status = None
# wait for the service to start the task
while not self.cli.tasks(filters={'service': self.service['ID']}):
continue
while True:
status = self.cli.tasks(
filters={'service': self.service['ID']})[0]['Status']['State']
if status in ['failed', 'complete']:
self.log.info('Service status before exiting: %s', status)
break
if self.auto_remove:
self.cli.remove_service(self.service['ID'])
if status == 'failed':
raise AirflowException('Service failed: ' + repr(self.service))
def scale_service_bk_2(self, target_service_name, replicas, arguments):
new_mode = {'replicated': {'Replicas': replicas}} #global
services = APIClient.services()
target_service = None
for service in services:
if service["Spec"]["Name"] == target_service_name:
session = requests_unixsocket.Session()
r1 = session.get('http+unix://' +
urllib.quote_plus('/var/run/docker.sock') +
'/v1.25/services/' + service["ID"])
#pprint.pprint(r1.json())
target_service = r1.json()
break
if not target_service is None:
task_template = types.TaskTemplate(service["Spec"]["TaskTemplate"])
old_constraints = service['Spec']['TaskTemplate']['Placement'][
'Constraints']
placement_constraints = Placement(old_constraints)
container_image = task_template['ContainerSpec']['ContainerSpec'][
'Image']
container_command = task_template['ContainerSpec'][
'ContainerSpec']['Command']
container_spec = ContainerSpec(image=container_image,
command=container_command,
args=arguments) #
# Issue between Docker engine API and docker Python API !!!
adapted_template = TaskTemplate(container_spec,
placement=placement_constraints)
networks = target_service["Spec"]['Networks']
print(adapted_template)
#print(replicas)
#print(json.dumps(target_service));
data = {
'Name': target_service_name,
'TaskTemplate': adapted_template,
'Mode': {
'Replicated': {
'Replicas': replicas
}
},
'Endpoint': service['Endpoint'],
'Networks': service['Spec']['Networks']
}
#print(json.dumps(data));
headers = {}
headers['Content-Type'] = 'application/json'
session = requests_unixsocket.Session()
r = session.post(
'http+unix://' + urllib.quote_plus('/var/run/docker.sock') +
'/v1.25/services/' + str(target_service["ID"]) +
'/update?version=' + str(target_service["Version"]["Index"]),
json=data,
headers=headers)
pprint.pprint(r.json())
else:
print "can't scale service :" + str(target_service_name)
def _run_service(self) -> None:
self.log.info('Starting docker service from image %s', self.image)
if not self.cli:
raise Exception("The 'cli' should be initialized before!")
self.service = self.cli.create_service(
types.TaskTemplate(
container_spec=types.ContainerSpec(
image=self.image,
command=self.format_command(self.command),
mounts=self.mounts,
env=self.environment,
user=self.user,
tty=self.tty,
configs=self.configs,
secrets=self.secrets,
),
restart_policy=types.RestartPolicy(condition='none'),
resources=types.Resources(mem_limit=self.mem_limit),
networks=self.networks,
placement=self.placement,
),
name=f'airflow-{get_random_string()}',
labels={'name': f'airflow__{self.dag_id}__{self.task_id}'},
mode=self.mode,
)
if self.service is None:
raise Exception("Service should be set here")
self.log.info('Service started: %s', str(self.service))
# wait for the service to start the task
while not self.cli.tasks(filters={'service': self.service['ID']}):
continue
if self.enable_logging:
self._stream_logs_to_output()
while True:
if self._has_service_terminated():
self.log.info('Service status before exiting: %s',
self._service_status())
break
if self.service and self._service_status() != 'complete':
if self.auto_remove:
self.cli.remove_service(self.service['ID'])
raise AirflowException('Service did not complete: ' +
repr(self.service))
elif self.auto_remove:
if not self.service:
raise Exception("The 'service' should be initialized before!")
self.cli.remove_service(self.service['ID'])
def _create_service(self):
return self.cli.create_service(
types.TaskTemplate(
container_spec=types.ContainerSpec(
image=self.image,
command=self.get_command(),
env=self.environment,
user=self.user,
tty=self.tty,
configs=self._get_configs(),
secrets=self._get_secrets(),
),
restart_policy=types.RestartPolicy(condition='none'),
resources=types.Resources(mem_limit=self.mem_limit),
networks=self.networks,
),
name=self._get_service_name(),
labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)})
def _run_service(self) -> None:
self.log.info('Starting docker service from image %s', self.image)
if not self.cli:
raise Exception("The 'cli' should be initialized before!")
self.service = self.cli.create_service(
types.TaskTemplate(
container_spec=types.ContainerSpec(
image=self.image,
command=self.get_command(),
env=self.environment,
user=self.user,
tty=self.tty,
),
restart_policy=types.RestartPolicy(condition='none'),
resources=types.Resources(mem_limit=self.mem_limit),
),
name='airflow-%s' % get_random_string(),
labels={'name': 'airflow__%s__%s' % (self.dag_id, self.task_id)},
)
self.log.info('Service started: %s', str(self.service))
# wait for the service to start the task
while not self.cli.tasks(filters={'service': self.service['ID']}):
continue
if self.enable_logging:
self._stream_logs_to_output()
while True:
if self._has_service_terminated():
self.log.info('Service status before exiting: %s',
self._service_status())
break
if self.auto_remove:
if not self.service:
raise Exception("The 'service' should be initialized before!")
self.cli.remove_service(self.service['ID'])
if self._service_status() == 'failed':
raise AirflowException('Service failed: ' + repr(self.service))
def generate_docker_py_service_description(self, name, docker_networks):
mounts = []
for mount_config in self.mounts:
mounts.append(
types.Mount(target=mount_config['target'],
source=mount_config['source'],
type=mount_config['type'],
read_only=mount_config['readonly']))
configs = []
for config_config in self.configs:
configs.append(
types.ConfigReference(config_id=config_config['config_id'],
config_name=config_config['config_name'],
filename=config_config.get('filename'),
uid=config_config.get('uid'),
gid=config_config.get('gid'),
mode=config_config.get('mode')))
secrets = []
for secret_config in self.secrets:
secrets.append(
types.SecretReference(secret_id=secret_config['secret_id'],
secret_name=secret_config['secret_name'],
filename=secret_config.get('filename'),
uid=secret_config.get('uid'),
gid=secret_config.get('gid'),
mode=secret_config.get('mode')))
cspec = types.ContainerSpec(image=self.image,
user=self.user,
dns_config=types.DNSConfig(
nameservers=self.dns,
search=self.dns_search,
options=self.dns_options),
args=self.args,
env=self.env,
tty=self.tty,
hostname=self.hostname,
labels=self.container_labels,
mounts=mounts,
secrets=secrets,
configs=configs)
log_driver = types.DriverConfig(name=self.log_driver,
options=self.log_driver_options)
placement = types.Placement(constraints=self.constraints)
restart_policy = types.RestartPolicy(
condition=self.restart_policy,
delay=self.restart_policy_delay,
max_attempts=self.restart_policy_attempts,
window=self.restart_policy_window)
resources = types.Resources(
cpu_limit=int(self.limit_cpu * 1000000000.0),
mem_limit=self.limit_memory,
cpu_reservation=int(self.reserve_cpu * 1000000000.0),
mem_reservation=self.reserve_memory)
update_policy = types.UpdateConfig(
parallelism=self.update_parallelism,
delay=self.update_delay,
failure_action=self.update_failure_action,
monitor=self.update_monitor,
max_failure_ratio=self.update_max_failure_ratio,
order=self.update_order)
task_template = types.TaskTemplate(container_spec=cspec,
log_driver=log_driver,
restart_policy=restart_policy,
placement=placement,
resources=resources,
force_update=self.force_update)
if self.mode == 'global':
self.replicas = None
mode = types.ServiceMode(self.mode, replicas=self.replicas)
networks = []
for network_name in self.networks:
network_id = None
try:
network_id = list(
filter(lambda n: n['name'] == network_name,
docker_networks))[0]['id']
except:
pass
if network_id:
networks.append({'Target': network_id})
else:
raise Exception("no docker networks named: %s" % network_name)
ports = {}
for port in self.publish:
ports[int(port['published_port'])] = (int(port['target_port']),
port['protocol'],
port['mode'])
endpoint_spec = types.EndpointSpec(mode=self.endpoint_mode,
ports=ports)
return update_policy, task_template, networks, endpoint_spec, mode, self.labels
