def _record_state(self, client, instance, docker_id=None):
if docker_id is None:
container = self.get_container(client, instance)
if container is not None:
docker_id = container['Id']
if docker_id is None:
return
cont_dir = Config.container_state_dir()
tmp_file_path = path.join(cont_dir, 'tmp-%s' % docker_id)
if path.exists(tmp_file_path):
remove(tmp_file_path)
file_path = path.join(cont_dir, docker_id)
if path.exists(file_path):
remove(file_path)
if not path.exists(cont_dir):
makedirs(cont_dir)
with open(tmp_file_path, 'w') as outfile:
marshaller = get_type(MARSHALLER)
data = marshaller.to_string(instance)
outfile.write(data)
rename(tmp_file_path, file_path)
def before_start(self, instance, host, progress, config):
try:
metadata = instance.data.metadata['meta-data']
userdata = instance.data.metadata['user-data']
except (KeyError, AttributeError):
return
log.info('Setting up config drive for %s', instance.uuid)
os_metadata = self._create_os_meta_data(instance, metadata)
if userdata is None:
userdata = ''
self._filter_meta_data(metadata)
marshaller = get_type(MARSHALLER)
meta_data_content = marshaller.to_string(metadata)
os_meta_data_content = marshaller.to_string(os_metadata)
content = [
('ec2/2009-04-04/meta-data.json', meta_data_content),
('ec2/2009-04-04/user-data', userdata),
('ec2/latest/meta-data.json', meta_data_content),
('ec2/latest/user-data', userdata,),
('openstack/2012-08-10/meta_data.json', os_meta_data_content),
('openstack/2012-08-10/user_data', userdata,),
('openstack/latest/meta_data.json', os_meta_data_content),
('openstack/latest/user_data', userdata)
]
hash = self._hash_content(content)
iso_file = self._write_iso(instance, content, hash)
config.set_param('config-drive-iso', iso_file)
def _record_state(self, client, instance, docker_id=None):
if docker_id is None:
container = self.get_container(client, instance)
if container is not None:
docker_id = container['Id']
if docker_id is None:
return
cont_dir = Config.container_state_dir()
tmp_file_path = path.join(cont_dir, 'tmp-%s' % docker_id)
if path.exists(tmp_file_path):
remove(tmp_file_path)
file_path = path.join(cont_dir, docker_id)
if path.exists(file_path):
remove(file_path)
if not path.exists(cont_dir):
makedirs(cont_dir)
with open(tmp_file_path, 'w') as outfile:
marshaller = get_type(MARSHALLER)
data = marshaller.to_string(instance)
outfile.write(data)
rename(tmp_file_path, file_path)
def _worker_main(worker_name, queue, ppid):
agent = Agent()
marshaller = type_manager.get_type(type_manager.MARSHALLER)
publisher = type_manager.get_type(type_manager.PUBLISHER)
while True:
try:
req = None
line = queue.get(True, 5)
req = marshaller.from_string(line)
utils.log_request(req, log, 'Request: %s', line)
id = req.id
start = time.time()
try:
utils.log_request(req, log, '%s : Starting request %s for %s',
worker_name, id, req.name)
resp = agent.execute(req)
if resp is not None:
publisher.publish(resp)
finally:
duration = time.time() - start
utils.log_request(req, log,
'%s : Done request %s for %s [%s] seconds',
worker_name, id, req.name, duration)
except Empty:
if not _should_run(ppid):
break
except FailedToLock as e:
log.info("%s for %s", e, req.name)
if not _should_run(ppid):
break
except Exception as e:
error_id = str(uuid.uuid4())
log.exception("%s : Unknown error", error_id)
if not _should_run(ppid):
break
if req is not None:
msg = "{0} : {1}".format(error_id, e)
resp = utils.reply(req)
if resp is not None:
resp["transitioning"] = "error"
resp["transitioningInternalMessage"] = msg
publisher.publish(resp)
def ns_exec(pid, event):
script = os.path.join(Config.home(), 'events', event.name.split(';')[0])
cmd = ['nsenter',
'-F',
'-m',
'-u',
'-i',
'-n',
'-p',
'-t', str(pid),
'--', script]
marshaller = get_type(MARSHALLER)
input = marshaller.to_string(event)
data = None
env = {}
#in come customized docker like alidocker use with open('/host/proc/{}/environ'.format(pid)) as f:
with open('/proc/{}/environ'.format(pid)) as f:
for line in f.read().split('\0'):
if not len(line):
continue
kv = line.split('=', 1)
if kv[0].startswith('CATTLE'):
env[kv[0]] = kv[1]
env['PATH'] = os.environ['PATH']
env['CATTLE_CONFIG_URL'] = Config.config_url()
for i in range(3):
p = popen(cmd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = p.communicate(input=input)
retcode = p.poll()
if retcode == 0:
break
exists_cmd = cmd[:-1] + ['/usr/bin/test', '-e', script]
if popen(exists_cmd, env=env).wait() == 0:
break
# Sleep and try again if missing
time.sleep(1)
if retcode:
return retcode, output, None
text = []
for line in output.splitlines():
if line.startswith('{'):
data = marshaller.from_string(line)
break
text.append(line)
return retcode, ''.join(text), data
def ns_exec(pid, event):
script = os.path.join(Config.home(), 'events', event.name.split(';')[0])
cmd = ['nsenter',
'-F',
'-m',
'-u',
'-i',
'-n',
'-p',
'-t', str(pid),
'--', script]
marshaller = get_type(MARSHALLER)
input = marshaller.to_string(event)
data = None
env = {}
with open('/proc/{}/environ'.format(pid)) as f:
for line in f.read().split('\0'):
if not len(line):
continue
kv = line.split('=', 1)
if kv[0].startswith('CATTLE'):
env[kv[0]] = kv[1]
env['PATH'] = os.environ['PATH']
env['CATTLE_CONFIG_URL'] = Config.config_url()
for i in range(3):
p = popen(cmd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = p.communicate(input=input)
retcode = p.poll()
if retcode == 0:
break
exists_cmd = cmd[:-1] + ['/usr/bin/test', '-e', script]
if popen(exists_cmd, env=env).wait() == 0:
break
# Sleep and try again if missing
time.sleep(1)
if retcode:
return retcode, output, None
text = []
for line in output.splitlines():
if line.startswith('{'):
data = marshaller.from_string(line)
break
text.append(line)
return retcode, ''.join(text), data
def _do_image_activate(self, image, storage_pool, progress):
if is_no_op(image):
return
if self._is_build(image):
return self._image_build(image, progress)
auth_config = None
try:
if "registryCredential" in image:
if image.registryCredential is not None:
auth_config = {
"username": image.registryCredential["publicValue"],
"email": image.registryCredential["data"]["fields"]["email"],
"password": image.registryCredential["secretValue"],
"serveraddress": image.registryCredential["registry"]["data"]["fields"]["serverAddress"],
}
if auth_config["serveraddress"] == "https://docker.io":
auth_config["serveraddress"] = "https://index.docker.io"
log.debug("Auth_Config: [%s]", auth_config)
else:
log.debug("No Registry credential found. Pulling non-authed")
except (AttributeError, KeyError, TypeError) as e:
raise AuthConfigurationError(
"Malformed Auth Config. \n\n" "error: [%s]\nregistryCredential:" " %s" % (e, image.registryCredential)
)
client = docker_client()
data = image.data.dockerImage
marshaller = get_type(MARSHALLER)
temp = data.qualifiedName
if data.qualifiedName.startswith("docker.io/"):
temp = "index." + data.qualifiedName
# Always pass insecure_registry=True to prevent docker-py
# from pre-verifying the registry. Let the docker daemon handle
# the verification of and connection to the registry.
if progress is None:
result = client.pull(repository=temp, tag=data.tag, auth_config=auth_config, insecure_registry=True)
if "error" in result:
raise ImageValidationError("Image [%s] failed to pull: %s" % (data.fullName, result["error"]))
else:
last_message = ""
message = ""
for status in client.pull(
repository=temp, tag=data.tag, auth_config=auth_config, stream=True, insecure_registry=True
):
try:
status = marshaller.from_string(status)
if "error" in status:
message = status["error"]
raise ImageValidationError("Image [%s] failed to pull:" " %s" % (data.fullName, message))
if "status" in status:
message = status["status"]
except ImageValidationError, e:
raise e
except:
# Ignore errors reading the status from Docker
continue
def do_build():
for key in ['context', 'remote']:
if key in opts:
del opts[key]
opts['stream'] = True
marshaller = get_type(MARSHALLER)
for status in client.build(**opts):
try:
status = marshaller.from_string(status)
progress.update(status['stream'])
except:
pass
def do_build():
for key in ['context', 'remote']:
if key in opts:
del opts[key]
opts['stream'] = True
marshaller = get_type(MARSHALLER)
for status in client.build(**opts):
try:
status = marshaller.from_string(status)
progress.update(status['stream'])
except:
pass
def _do_image_activate(self, image, storage_pool, progress):
if is_no_op(image):
return
auth_config = None
try:
if 'registryCredential' in image:
if image.registryCredential is not None:
auth_config = {
'username': image.registryCredential['publicValue'],
'email': image.registryCredential['data']['fields']
['email'],
'password': image.registryCredential['secretValue'],
'serveraddress': image.registryCredential['registry']
['data']['fields']['serverAddress']
}
log.debug('Auth_Config: [%s]', auth_config)
else:
log.debug('No Registry credential found. Pulling non-authed')
except (AttributeError, KeyError, TypeError) as e:
raise AuthConfigurationError("Malformed Auth Config. \n\n"
"error: [%s]\nregistryCredential:"
" %s"
% (e, image.registryCredential))
client = docker_client()
data = image.data.dockerImage
marshaller = get_type(MARSHALLER)
if progress is None:
result = client.pull(repository=data.qualifiedName,
tag=data.tag, auth_config=auth_config)
if 'error' in result:
raise ImageValidationError('Image [%s] failed to pull' %
data.fullName)
else:
for status in client.pull(repository=data.qualifiedName,
tag=data.tag,
auth_config=auth_config,
stream=True):
log.info('Pulling [%s] status : %s', data.fullName, status)
status = marshaller.from_string(status)
try:
message = status['status']
except KeyError:
message = status['error']
raise ImageValidationError('Image [%s] failed to pull '
': %s' % (data.fullName,
message))
progress.update(message)
def ns_exec(pid, event):
script = os.path.join(Config.home(), 'events', event.name.split(';')[0])
cmd = ['nsenter',
'-F',
'-m',
'-u',
'-i',
'-n',
'-p',
'-t', str(pid),
'--', script]
marshaller = get_type(MARSHALLER)
input = marshaller.to_string(event)
data = None
env = {}
with open('/proc/{}/environ'.format(pid)) as f:
for line in f.read().split('\0'):
if not len(line):
continue
kv = line.split('=', 1)
if kv[0].startswith('CATTLE'):
env[kv[0]] = kv[1]
env['PATH'] = os.environ['PATH']
p = popen(cmd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = p.communicate(input=input)
retcode = p.poll()
if retcode:
return retcode, output, None
text = []
for line in output.splitlines():
if line.startswith('{'):
data = marshaller.from_string(line)
break
text.append(line)
return retcode, ''.join(text), data
def update(self, msg, progress=None, data=None):
resp = utils.reply(self._req, data)
resp['transitioning'] = 'yes'
resp['transitioningMessage'] = msg
resp['transitioningProgress'] = progress
if self._parent is not None:
resp = utils.reply(self._parent, resp)
resp['transitioning'] = 'yes'
resp['transitioningMessage'] = msg
resp['transitioningProgress'] = progress
publisher = get_type(PUBLISHER)
try:
publisher.publish(resp)
except:
pass
def _do_image_activate(self, image, storage_pool, progress):
client = docker_client()
data = image.data.dockerImage
marshaller = get_type(MARSHALLER)
if progress is None:
client.pull(repository=data.qualifiedName, tag=data.tag)
else:
for status in client.pull(repository=data.qualifiedName,
tag=data.tag,
stream=True):
try:
log.info('Pulling [%s] status : %s', data.fullName, status)
status = marshaller.from_string(status)
message = status['status']
progress.update(message)
except:
pass
def container_exec(ip, token, event):
marshaller = get_type(MARSHALLER)
data = marshaller.to_string(event)
url = 'http://{0}:8080/events?token={1}'.format(ip, token)
r = _SESSION.post(url, data=data, headers={
'Content-Type': 'application/json'
}, timeout=DockerConfig.delegate_timeout())
if r.status_code != 200:
return r.status_code, r.text, None
result = r.json()
data = result.get('data')
if data is not None:
data = marshaller.from_string(data)
return result.get('exitCode'), result.get('output'), data
def container_exec(ip, token, event):
marshaller = get_type(MARSHALLER)
data = marshaller.to_string(event)
url = 'http://{0}:8080/events?token={1}'.format(ip, token)
r = _SESSION.post(url,
data=data,
headers={'Content-Type': 'application/json'},
timeout=DockerConfig.delegate_timeout())
if r.status_code != 200:
return r.status_code, r.text, None
result = r.json()
data = result.get('data')
if data is not None:
data = marshaller.from_string(data)
return result.get('exitCode'), result.get('output'), data
def ns_exec(pid, event):
script = os.path.join(Config.home(), 'events', event.name.split(';')[0])
cmd = [
'nsenter', '-F', '-m', '-u', '-i', '-n', '-p', '-t',
str(pid), '--', script
]
marshaller = get_type(MARSHALLER)
input = marshaller.to_string(event)
data = None
env = {}
with open('/proc/{}/environ'.format(pid)) as f:
for line in f.read().split('\0'):
if not len(line):
continue
kv = line.split('=', 1)
if kv[0].startswith('CATTLE'):
env[kv[0]] = kv[1]
env['PATH'] = os.environ['PATH']
p = popen(cmd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = p.communicate(input=input)
retcode = p.poll()
if retcode:
return retcode, output, None
text = []
for line in output.splitlines():
if line.startswith('{'):
data = marshaller.from_string(line)
break
text.append(line)
return retcode, ''.join(text), data
def _do_image_activate(self, image, storage_pool, progress):
if is_no_op(image):
return
if self._is_build(image):
return self._image_build(image, progress)
auth_config = None
try:
if 'registryCredential' in image:
if image.registryCredential is not None:
auth_config = {
'username': image.registryCredential['publicValue'],
'email': image.registryCredential['data']['fields']
['email'],
'password': image.registryCredential['secretValue'],
'serveraddress': image.registryCredential['registry']
['data']['fields']['serverAddress']
}
if auth_config['serveraddress'] == "https://docker.io":
auth_config['serveraddress'] =\
"https://index.docker.io"
log.debug('Auth_Config: [%s]', auth_config)
else:
log.debug('No Registry credential found. Pulling non-authed')
except (AttributeError, KeyError, TypeError) as e:
raise AuthConfigurationError("Malformed Auth Config. \n\n"
"error: [%s]\nregistryCredential:"
" %s"
% (e, image.registryCredential))
client = docker_client()
data = image.data.dockerImage
marshaller = get_type(MARSHALLER)
temp = data.qualifiedName
if data.qualifiedName.startswith('docker.io/'):
temp = 'index.' + data.qualifiedName
# Always pass insecure_registry=True to prevent docker-py
# from pre-verifying the registry. Let the docker daemon handle
# the verification of and connection to the registry.
if progress is None:
result = client.pull(repository=temp,
tag=data.tag, auth_config=auth_config,
insecure_registry=True)
if 'error' in result:
raise ImageValidationError('Image [%s] failed to pull' %
data.fullName)
else:
for status in client.pull(repository=temp,
tag=data.tag,
auth_config=auth_config,
stream=True,
insecure_registry=True):
status = marshaller.from_string(status)
try:
message = status['status']
except KeyError:
message = status['error']
raise ImageValidationError('Image [%s] failed to pull '
': %s' % (data.fullName,
message))
progress.update(message)
def __init__(self, url, auth):
self._url = url
self._auth = auth
self._marshaller = type_manager.get_type(type_manager.MARSHALLER)
self._session = requests.Session()
def json_data(name):
marshaller = type_manager.get_type(type_manager.MARSHALLER)
with open(os.path.join(TEST_DIR, name)) as f:
return marshaller.from_string(f.read())
def json_data(name):
marshaller = type_manager.get_type(type_manager.MARSHALLER)
with open(os.path.join(TEST_DIR, name)) as f:
return marshaller.from_string(f.read())
def _do_image_activate(self, image, storage_pool, progress):
if is_no_op(image):
return
if self._is_build(image):
return self._image_build(image, progress)
auth_config = None
try:
if 'registryCredential' in image:
if image.registryCredential is not None:
auth_config = {
'username':
image.registryCredential['publicValue'],
'email':
image.registryCredential['data']['fields']['email'],
'password':
image.registryCredential['secretValue'],
'serveraddress':
image.registryCredential['registry']['data']['fields']
['serverAddress']
}
if auth_config['serveraddress'] == "https://docker.io":
auth_config['serveraddress'] =\
"https://index.docker.io"
log.debug('Auth_Config: [%s]', auth_config)
else:
log.debug('No Registry credential found. Pulling non-authed')
except (AttributeError, KeyError, TypeError) as e:
raise AuthConfigurationError("Malformed Auth Config. \n\n"
"error: [%s]\nregistryCredential:"
" %s" % (e, image.registryCredential))
client = docker_client()
data = image.data.dockerImage
marshaller = get_type(MARSHALLER)
temp = data.qualifiedName
if data.qualifiedName.startswith('docker.io/'):
temp = 'index.' + data.qualifiedName
# Always pass insecure_registry=True to prevent docker-py
# from pre-verifying the registry. Let the docker daemon handle
# the verification of and connection to the registry.
if progress is None:
result = client.pull(repository=temp,
tag=data.tag,
auth_config=auth_config,
insecure_registry=True)
if 'error' in result:
raise ImageValidationError('Image [%s] failed to pull: %s' %
(data.fullName, result['error']))
else:
last_message = ''
message = ''
for status in client.pull(repository=temp,
tag=data.tag,
auth_config=auth_config,
stream=True,
insecure_registry=True):
try:
status = marshaller.from_string(status)
if 'error' in status:
message = status['error']
raise ImageValidationError('Image [%s] failed to pull:'
' %s' %
(data.fullName, message))
if 'status' in status:
message = status['status']
except ImageValidationError, e:
raise e
except:
# Ignore errors reading the status from Docker
continue
