def _record_state(self, client, instance, docker_id=None): if docker_id is None: container = self.get_container(client, instance) if container is not None: docker_id = container['Id'] if docker_id is None: return cont_dir = Config.container_state_dir() tmp_file_path = path.join(cont_dir, 'tmp-%s' % docker_id) if path.exists(tmp_file_path): remove(tmp_file_path) file_path = path.join(cont_dir, docker_id) if path.exists(file_path): remove(file_path) if not path.exists(cont_dir): makedirs(cont_dir) with open(tmp_file_path, 'w') as outfile: marshaller = get_type(MARSHALLER) data = marshaller.to_string(instance) outfile.write(data) rename(tmp_file_path, file_path)
def before_start(self, instance, host, progress, config): try: metadata = instance.data.metadata['meta-data'] userdata = instance.data.metadata['user-data'] except (KeyError, AttributeError): return log.info('Setting up config drive for %s', instance.uuid) os_metadata = self._create_os_meta_data(instance, metadata) if userdata is None: userdata = '' self._filter_meta_data(metadata) marshaller = get_type(MARSHALLER) meta_data_content = marshaller.to_string(metadata) os_meta_data_content = marshaller.to_string(os_metadata) content = [ ('ec2/2009-04-04/meta-data.json', meta_data_content), ('ec2/2009-04-04/user-data', userdata), ('ec2/latest/meta-data.json', meta_data_content), ('ec2/latest/user-data', userdata,), ('openstack/2012-08-10/meta_data.json', os_meta_data_content), ('openstack/2012-08-10/user_data', userdata,), ('openstack/latest/meta_data.json', os_meta_data_content), ('openstack/latest/user_data', userdata) ] hash = self._hash_content(content) iso_file = self._write_iso(instance, content, hash) config.set_param('config-drive-iso', iso_file)
def _worker_main(worker_name, queue, ppid): agent = Agent() marshaller = type_manager.get_type(type_manager.MARSHALLER) publisher = type_manager.get_type(type_manager.PUBLISHER) while True: try: req = None line = queue.get(True, 5) req = marshaller.from_string(line) utils.log_request(req, log, 'Request: %s', line) id = req.id start = time.time() try: utils.log_request(req, log, '%s : Starting request %s for %s', worker_name, id, req.name) resp = agent.execute(req) if resp is not None: publisher.publish(resp) finally: duration = time.time() - start utils.log_request(req, log, '%s : Done request %s for %s [%s] seconds', worker_name, id, req.name, duration) except Empty: if not _should_run(ppid): break except FailedToLock as e: log.info("%s for %s", e, req.name) if not _should_run(ppid): break except Exception as e: error_id = str(uuid.uuid4()) log.exception("%s : Unknown error", error_id) if not _should_run(ppid): break if req is not None: msg = "{0} : {1}".format(error_id, e) resp = utils.reply(req) if resp is not None: resp["transitioning"] = "error" resp["transitioningInternalMessage"] = msg publisher.publish(resp)
def ns_exec(pid, event): script = os.path.join(Config.home(), 'events', event.name.split(';')[0]) cmd = ['nsenter', '-F', '-m', '-u', '-i', '-n', '-p', '-t', str(pid), '--', script] marshaller = get_type(MARSHALLER) input = marshaller.to_string(event) data = None env = {} #in come customized docker like alidocker use with open('/host/proc/{}/environ'.format(pid)) as f: with open('/proc/{}/environ'.format(pid)) as f: for line in f.read().split('\0'): if not len(line): continue kv = line.split('=', 1) if kv[0].startswith('CATTLE'): env[kv[0]] = kv[1] env['PATH'] = os.environ['PATH'] env['CATTLE_CONFIG_URL'] = Config.config_url() for i in range(3): p = popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = p.communicate(input=input) retcode = p.poll() if retcode == 0: break exists_cmd = cmd[:-1] + ['/usr/bin/test', '-e', script] if popen(exists_cmd, env=env).wait() == 0: break # Sleep and try again if missing time.sleep(1) if retcode: return retcode, output, None text = [] for line in output.splitlines(): if line.startswith('{'): data = marshaller.from_string(line) break text.append(line) return retcode, ''.join(text), data
def ns_exec(pid, event): script = os.path.join(Config.home(), 'events', event.name.split(';')[0]) cmd = ['nsenter', '-F', '-m', '-u', '-i', '-n', '-p', '-t', str(pid), '--', script] marshaller = get_type(MARSHALLER) input = marshaller.to_string(event) data = None env = {} with open('/proc/{}/environ'.format(pid)) as f: for line in f.read().split('\0'): if not len(line): continue kv = line.split('=', 1) if kv[0].startswith('CATTLE'): env[kv[0]] = kv[1] env['PATH'] = os.environ['PATH'] env['CATTLE_CONFIG_URL'] = Config.config_url() for i in range(3): p = popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = p.communicate(input=input) retcode = p.poll() if retcode == 0: break exists_cmd = cmd[:-1] + ['/usr/bin/test', '-e', script] if popen(exists_cmd, env=env).wait() == 0: break # Sleep and try again if missing time.sleep(1) if retcode: return retcode, output, None text = [] for line in output.splitlines(): if line.startswith('{'): data = marshaller.from_string(line) break text.append(line) return retcode, ''.join(text), data
def _do_image_activate(self, image, storage_pool, progress): if is_no_op(image): return if self._is_build(image): return self._image_build(image, progress) auth_config = None try: if "registryCredential" in image: if image.registryCredential is not None: auth_config = { "username": image.registryCredential["publicValue"], "email": image.registryCredential["data"]["fields"]["email"], "password": image.registryCredential["secretValue"], "serveraddress": image.registryCredential["registry"]["data"]["fields"]["serverAddress"], } if auth_config["serveraddress"] == "https://docker.io": auth_config["serveraddress"] = "https://index.docker.io" log.debug("Auth_Config: [%s]", auth_config) else: log.debug("No Registry credential found. Pulling non-authed") except (AttributeError, KeyError, TypeError) as e: raise AuthConfigurationError( "Malformed Auth Config. \n\n" "error: [%s]\nregistryCredential:" " %s" % (e, image.registryCredential) ) client = docker_client() data = image.data.dockerImage marshaller = get_type(MARSHALLER) temp = data.qualifiedName if data.qualifiedName.startswith("docker.io/"): temp = "index." + data.qualifiedName # Always pass insecure_registry=True to prevent docker-py # from pre-verifying the registry. Let the docker daemon handle # the verification of and connection to the registry. if progress is None: result = client.pull(repository=temp, tag=data.tag, auth_config=auth_config, insecure_registry=True) if "error" in result: raise ImageValidationError("Image [%s] failed to pull: %s" % (data.fullName, result["error"])) else: last_message = "" message = "" for status in client.pull( repository=temp, tag=data.tag, auth_config=auth_config, stream=True, insecure_registry=True ): try: status = marshaller.from_string(status) if "error" in status: message = status["error"] raise ImageValidationError("Image [%s] failed to pull:" " %s" % (data.fullName, message)) if "status" in status: message = status["status"] except ImageValidationError, e: raise e except: # Ignore errors reading the status from Docker continue
def do_build(): for key in ['context', 'remote']: if key in opts: del opts[key] opts['stream'] = True marshaller = get_type(MARSHALLER) for status in client.build(**opts): try: status = marshaller.from_string(status) progress.update(status['stream']) except: pass
def _do_image_activate(self, image, storage_pool, progress): if is_no_op(image): return auth_config = None try: if 'registryCredential' in image: if image.registryCredential is not None: auth_config = { 'username': image.registryCredential['publicValue'], 'email': image.registryCredential['data']['fields'] ['email'], 'password': image.registryCredential['secretValue'], 'serveraddress': image.registryCredential['registry'] ['data']['fields']['serverAddress'] } log.debug('Auth_Config: [%s]', auth_config) else: log.debug('No Registry credential found. Pulling non-authed') except (AttributeError, KeyError, TypeError) as e: raise AuthConfigurationError("Malformed Auth Config. \n\n" "error: [%s]\nregistryCredential:" " %s" % (e, image.registryCredential)) client = docker_client() data = image.data.dockerImage marshaller = get_type(MARSHALLER) if progress is None: result = client.pull(repository=data.qualifiedName, tag=data.tag, auth_config=auth_config) if 'error' in result: raise ImageValidationError('Image [%s] failed to pull' % data.fullName) else: for status in client.pull(repository=data.qualifiedName, tag=data.tag, auth_config=auth_config, stream=True): log.info('Pulling [%s] status : %s', data.fullName, status) status = marshaller.from_string(status) try: message = status['status'] except KeyError: message = status['error'] raise ImageValidationError('Image [%s] failed to pull ' ': %s' % (data.fullName, message)) progress.update(message)
def ns_exec(pid, event): script = os.path.join(Config.home(), 'events', event.name.split(';')[0]) cmd = ['nsenter', '-F', '-m', '-u', '-i', '-n', '-p', '-t', str(pid), '--', script] marshaller = get_type(MARSHALLER) input = marshaller.to_string(event) data = None env = {} with open('/proc/{}/environ'.format(pid)) as f: for line in f.read().split('\0'): if not len(line): continue kv = line.split('=', 1) if kv[0].startswith('CATTLE'): env[kv[0]] = kv[1] env['PATH'] = os.environ['PATH'] p = popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = p.communicate(input=input) retcode = p.poll() if retcode: return retcode, output, None text = [] for line in output.splitlines(): if line.startswith('{'): data = marshaller.from_string(line) break text.append(line) return retcode, ''.join(text), data
def update(self, msg, progress=None, data=None): resp = utils.reply(self._req, data) resp['transitioning'] = 'yes' resp['transitioningMessage'] = msg resp['transitioningProgress'] = progress if self._parent is not None: resp = utils.reply(self._parent, resp) resp['transitioning'] = 'yes' resp['transitioningMessage'] = msg resp['transitioningProgress'] = progress publisher = get_type(PUBLISHER) try: publisher.publish(resp) except: pass
def _do_image_activate(self, image, storage_pool, progress): client = docker_client() data = image.data.dockerImage marshaller = get_type(MARSHALLER) if progress is None: client.pull(repository=data.qualifiedName, tag=data.tag) else: for status in client.pull(repository=data.qualifiedName, tag=data.tag, stream=True): try: log.info('Pulling [%s] status : %s', data.fullName, status) status = marshaller.from_string(status) message = status['status'] progress.update(message) except: pass
def container_exec(ip, token, event): marshaller = get_type(MARSHALLER) data = marshaller.to_string(event) url = 'http://{0}:8080/events?token={1}'.format(ip, token) r = _SESSION.post(url, data=data, headers={ 'Content-Type': 'application/json' }, timeout=DockerConfig.delegate_timeout()) if r.status_code != 200: return r.status_code, r.text, None result = r.json() data = result.get('data') if data is not None: data = marshaller.from_string(data) return result.get('exitCode'), result.get('output'), data
def container_exec(ip, token, event): marshaller = get_type(MARSHALLER) data = marshaller.to_string(event) url = 'http://{0}:8080/events?token={1}'.format(ip, token) r = _SESSION.post(url, data=data, headers={'Content-Type': 'application/json'}, timeout=DockerConfig.delegate_timeout()) if r.status_code != 200: return r.status_code, r.text, None result = r.json() data = result.get('data') if data is not None: data = marshaller.from_string(data) return result.get('exitCode'), result.get('output'), data
def ns_exec(pid, event): script = os.path.join(Config.home(), 'events', event.name.split(';')[0]) cmd = [ 'nsenter', '-F', '-m', '-u', '-i', '-n', '-p', '-t', str(pid), '--', script ] marshaller = get_type(MARSHALLER) input = marshaller.to_string(event) data = None env = {} with open('/proc/{}/environ'.format(pid)) as f: for line in f.read().split('\0'): if not len(line): continue kv = line.split('=', 1) if kv[0].startswith('CATTLE'): env[kv[0]] = kv[1] env['PATH'] = os.environ['PATH'] p = popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = p.communicate(input=input) retcode = p.poll() if retcode: return retcode, output, None text = [] for line in output.splitlines(): if line.startswith('{'): data = marshaller.from_string(line) break text.append(line) return retcode, ''.join(text), data
def _do_image_activate(self, image, storage_pool, progress): if is_no_op(image): return if self._is_build(image): return self._image_build(image, progress) auth_config = None try: if 'registryCredential' in image: if image.registryCredential is not None: auth_config = { 'username': image.registryCredential['publicValue'], 'email': image.registryCredential['data']['fields'] ['email'], 'password': image.registryCredential['secretValue'], 'serveraddress': image.registryCredential['registry'] ['data']['fields']['serverAddress'] } if auth_config['serveraddress'] == "https://docker.io": auth_config['serveraddress'] =\ "https://index.docker.io" log.debug('Auth_Config: [%s]', auth_config) else: log.debug('No Registry credential found. Pulling non-authed') except (AttributeError, KeyError, TypeError) as e: raise AuthConfigurationError("Malformed Auth Config. \n\n" "error: [%s]\nregistryCredential:" " %s" % (e, image.registryCredential)) client = docker_client() data = image.data.dockerImage marshaller = get_type(MARSHALLER) temp = data.qualifiedName if data.qualifiedName.startswith('docker.io/'): temp = 'index.' + data.qualifiedName # Always pass insecure_registry=True to prevent docker-py # from pre-verifying the registry. Let the docker daemon handle # the verification of and connection to the registry. if progress is None: result = client.pull(repository=temp, tag=data.tag, auth_config=auth_config, insecure_registry=True) if 'error' in result: raise ImageValidationError('Image [%s] failed to pull' % data.fullName) else: for status in client.pull(repository=temp, tag=data.tag, auth_config=auth_config, stream=True, insecure_registry=True): status = marshaller.from_string(status) try: message = status['status'] except KeyError: message = status['error'] raise ImageValidationError('Image [%s] failed to pull ' ': %s' % (data.fullName, message)) progress.update(message)
def __init__(self, url, auth): self._url = url self._auth = auth self._marshaller = type_manager.get_type(type_manager.MARSHALLER) self._session = requests.Session()
def json_data(name): marshaller = type_manager.get_type(type_manager.MARSHALLER) with open(os.path.join(TEST_DIR, name)) as f: return marshaller.from_string(f.read())
def _do_image_activate(self, image, storage_pool, progress): if is_no_op(image): return if self._is_build(image): return self._image_build(image, progress) auth_config = None try: if 'registryCredential' in image: if image.registryCredential is not None: auth_config = { 'username': image.registryCredential['publicValue'], 'email': image.registryCredential['data']['fields']['email'], 'password': image.registryCredential['secretValue'], 'serveraddress': image.registryCredential['registry']['data']['fields'] ['serverAddress'] } if auth_config['serveraddress'] == "https://docker.io": auth_config['serveraddress'] =\ "https://index.docker.io" log.debug('Auth_Config: [%s]', auth_config) else: log.debug('No Registry credential found. Pulling non-authed') except (AttributeError, KeyError, TypeError) as e: raise AuthConfigurationError("Malformed Auth Config. \n\n" "error: [%s]\nregistryCredential:" " %s" % (e, image.registryCredential)) client = docker_client() data = image.data.dockerImage marshaller = get_type(MARSHALLER) temp = data.qualifiedName if data.qualifiedName.startswith('docker.io/'): temp = 'index.' + data.qualifiedName # Always pass insecure_registry=True to prevent docker-py # from pre-verifying the registry. Let the docker daemon handle # the verification of and connection to the registry. if progress is None: result = client.pull(repository=temp, tag=data.tag, auth_config=auth_config, insecure_registry=True) if 'error' in result: raise ImageValidationError('Image [%s] failed to pull: %s' % (data.fullName, result['error'])) else: last_message = '' message = '' for status in client.pull(repository=temp, tag=data.tag, auth_config=auth_config, stream=True, insecure_registry=True): try: status = marshaller.from_string(status) if 'error' in status: message = status['error'] raise ImageValidationError('Image [%s] failed to pull:' ' %s' % (data.fullName, message)) if 'status' in status: message = status['status'] except ImageValidationError, e: raise e except: # Ignore errors reading the status from Docker continue