def _init(): flavor = os.environ.get('SETTINGS_FLAVOR', 'dev') config_path = os.environ.get('DOCKER_REGISTRY_CONFIG', 'config.yml') if not os.path.isabs(config_path): config_path = os.path.join(os.path.dirname(__file__), '../../', 'config', config_path) try: f = open(config_path) except Exception: raise exceptions.FileNotFoundError('Heads-up! File is missing: %s' % config_path) conf = Config(f.read()) if flavor: conf = conf[flavor] conf.flavor = flavor if conf.privileged_key: try: f = open(conf.privileged_key) except Exception: raise exceptions.FileNotFoundError( 'Heads-up! File is missing: %s' % conf.privileged_key) try: conf.privileged_key = rsa.PublicKey.load_pkcs1(f.read()) except Exception: raise exceptions.ConfigError('Key at %s is not a valid RSA key' % conf.privileged_key) if conf.index_endpoint: conf.index_endpoint = conf.index_endpoint.strip('/') return conf
def stream_read(self, path, bytes_range=None): path = self._init_path(path) nb_bytes = 0 total_size = 0 key = self._boto_bucket.lookup(path) if not key: raise exceptions.FileNotFoundError('%s is not there' % path) if bytes_range: key._last_position = bytes_range[0] total_size = bytes_range[1] - bytes_range[0] + 1 while True: if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = key.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = key.read(self.buffer_size) if not buf: break yield buf
def get_size(self, path): path = self._init_path(path) try: headers = self.head_store(path) return headers['size'] except Exception: raise exceptions.FileNotFoundError('%s is not there' % path)
def getfileinfo(self, path): logger.debug("getfileinfo: %s" % path) resp = self._retry(self._getfileinfo, path) if resp.status_code == 200: j = resp.json() key = "fragment-info" if key in j: fragementsinfo = j[key] max_range, fragementsinfo = self._check_fragments( fragementsinfo) if not max_range or not fragementsinfo: raise exceptions.UnspecifiedError("fragment-info Error!") return max_range, fragementsinfo else: raise exceptions.UnspecifiedError( "fileinfo not contain fragment-info!") elif resp.status_code == 404: raise exceptions.FileNotFoundError("File Not Found!") else: raise exceptions.UnspecifiedError( "getfileinfo UnKnow status code:%d" % resp.status_code)
def get_store(self, path, chunk_size=None): try: _, obj = self._swift_connection.get_object( self._swift_container, path, resp_chunk_size=chunk_size) return obj except Exception: raise exceptions.FileNotFoundError('%s is not there' % path)
def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path)
def _init(): flavor = os.environ.get('SETTINGS_FLAVOR', 'dev') config_path = os.environ.get('DOCKER_REGISTRY_CONFIG', 'config.yml') if not os.path.isabs(config_path): config_path = os.path.join(os.path.dirname(__file__), '../../', 'config', config_path) try: f = open(config_path) except Exception: raise exceptions.FileNotFoundError( 'Heads-up! File is missing: %s' % config_path) conf = Config(f.read()) if flavor: if flavor not in conf: raise exceptions.ConfigError( 'The specified flavor (%s) is missing in your config file (%s)' % (flavor, config_path)) conf = conf[flavor] conf.flavor = flavor if conf.privileged_key: try: f = open(conf.privileged_key) except Exception: raise exceptions.FileNotFoundError( 'Heads-up! File is missing: %s' % conf.privileged_key) try: pk = f.read().split('\n') pk = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A' + ''.join(pk[1:-2]) pk = [pk[i: i + 64] for i in range(0, len(pk), 64)] pk = ('-----BEGIN PUBLIC KEY-----\n' + '\n'.join(pk) + '\n-----END PUBLIC KEY-----') bio = BIO.MemoryBuffer(pk) conf.privileged_key = RSA.load_pub_key_bio(bio) except Exception: raise exceptions.ConfigError( 'Key at %s is not a valid RSA key' % conf.privileged_key) f.close() if conf.index_endpoint: conf.index_endpoint = conf.index_endpoint.strip('/') return conf
def s_read(self, path, offset=0, size=0): r = self._session.read_data(path, offset=offset, size=size) r.wait() err = r.error() if err.code != 0: raise exceptions.FileNotFoundError("No such file %s" % path) res = r.get()[0] return str(res.data)
def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs]
def _downloader(self, fragments): logger.debug("begin download multi parts.") for fragment in fragments: fragment_index = fragment['Index'] fragment_begin = fragment['Start'] fragment_end = fragment['End'] try: logger.debug("begin download part fragment_index: %d" % fragment_index) resp = self.conn.download(self.path, fragment_index, (fragment_begin, fragment_end), stream=False) if resp.status_code == 200: content = resp.content self._fragment_tempfiles[fragment_index] = _TempFile( mode='w+b', prefix=self.tmpdir) f = self._fragment_tempfiles[fragment_index].file f.write(content) # seek to 0, ready to be read f.seek(0) self._refresh_max_completed_byte(fragment_index, fragment_end) logger.debug( "download part success!!! fragment_index: %d" % fragment_index) elif resp.status_code == 404: logger.debug( "download part, file not found, path: %s, frament_index: %d" % (self.path, fragment_index)) raise exceptions.FileNotFoundError( "fetch_part FileNotFound!") else: logger.debug("mark else code: %d" % resp.status_code) raise exceptions.UnspecifiedError( "unexcept status code: %d" % resp.status_code) except: logger.error("download part failed, path: %s, index: %d" % (self.path, fragment_index)) self._set_error() return # docker read timeout now = time.time() if now - self._get_last_read_time() > 300: logger.debug( "speedy long time no read, reader maybe exited!!!") self._set_error() self.clear() return logger.debug("end download multi parts.")
def _get_image_layer(image_id, headers=None, bytes_range=None): if headers is None: headers = {} headers['Content-Type'] = 'application/octet-stream' accel_uri_prefix = cfg.nginx_x_accel_redirect path = store.image_layer_path(image_id) if accel_uri_prefix: if store.scheme == 'file': accel_uri = '/'.join([accel_uri_prefix, path]) headers['X-Accel-Redirect'] = accel_uri logger.debug('send accelerated {0} ({1})'.format( accel_uri, headers)) return flask.Response('', headers=headers) else: logger.warn('nginx_x_accel_redirect config set,' ' but storage is not LocalStorage') # If store allows us to just redirect the client let's do that, we'll # offload a lot of expensive I/O and get faster I/O if cfg.storage_redirect: try: content_redirect_url = store.content_redirect_url(path) if content_redirect_url: return flask.redirect(content_redirect_url, 302) except IOError as e: logger.debug(str(e)) status = None layer_size = 0 if not store.exists(path): raise exceptions.FileNotFoundError("Image layer absent from store") try: layer_size = store.get_size(path) except exceptions.FileNotFoundError: # XXX why would that fail given we know the layer exists? pass if bytes_range and bytes_range[1] == -1 and not layer_size == 0: bytes_range = (bytes_range[0], layer_size) if bytes_range: content_length = bytes_range[1] - bytes_range[0] + 1 if not _valid_bytes_range(bytes_range): return flask.Response(status=416, headers=headers) status = 206 content_range = (bytes_range[0], bytes_range[1], layer_size) headers['Content-Range'] = '{0}-{1}/{2}'.format(*content_range) headers['Content-Length'] = content_length elif layer_size > 0: headers['Content-Length'] = layer_size else: return flask.Response(status=416, headers=headers) return flask.Response(store.stream_read(path, bytes_range), headers=headers, status=status)
def list_directory(self, path=None): if path is None: # pragma: no cover path = "" if not self.exists(path) and path: raise exceptions.FileNotFoundError( 'No such directory: \'{0}\''.format(path)) for item in self.s_find(('docker', path)): yield item
def get_content(self, path): path = self.getfullpath(path) try: res = self._oss.get_object(self.osscfg.bucket, path) if res.status == 200: return res.read() else: raise IOError('read %s failed, status: %s' % (path, res.status)) except Exception: raise exceptions.FileNotFoundError("File not found %s" % path)
def get_content(self, path): if not self.exists(path): raise exceptions.FileNotFoundError("File not found %s" % path) path = self._init_path(path) output = StringIO.StringIO() try: for buf in self.get_store(path, self.buffer_size): output.write(buf) return output.getvalue() finally: output.close()
def load(): global _config if _config is not None: return _config data = None config_path = os.environ.get('DOCKER_REGISTRY_CONFIG', 'config.yml') if not os.path.isabs(config_path): config_path = os.path.join(os.path.dirname(__file__), '../../', 'config', config_path) try: f = open(config_path) except Exception: raise exceptions.FileNotFoundError( 'Heads-up! File is missing: %s' % config_path) try: data = yaml.load(f) except Exception: raise exceptions.ConfigError( 'Config file (%s) is not valid yaml' % config_path) config = data.get('common', {}) flavor = os.environ.get('SETTINGS_FLAVOR', 'dev') config.update(data.get(flavor, {})) config['flavor'] = flavor config = convert_env_vars(config) if 'privileged_key' in config: try: f = open(config['privileged_key']) except Exception: raise exceptions.FileNotFoundError( 'Heads-up! File is missing: %s' % config['privileged_key']) try: config['privileged_key'] = rsa.PublicKey.load_pkcs1(f.read()) except Exception: raise exceptions.ConfigError( 'Key at %s is not a valid RSA key' % config['privileged_key']) _config = Config(config) return _config
def stream_read(self, path, bytes_range=None): logger.debug("read range %s from %s", str(bytes_range), path) if not self.exists(path): raise exceptions.FileNotFoundError( 'No such directory: \'{0}\''.format(path)) if bytes_range is None: yield self.s_read(path) else: offset = bytes_range[0] size = bytes_range[1] - bytes_range[0] + 1 yield self.s_read(path, offset=offset, size=size)
def get_size(self, path): logger.debug("get_size of %s", path) r = self._session.lookup(path) r.wait() lookups = r.get() err = r.error() if err.code != 0: raise exceptions.FileNotFoundError("Unable to get size of %s %s" % (path, err)) size = lookups[0].size logger.debug("size of %s = %d", path, size) return size
def list_directory(self, path=None): logger.debug("list directory path: %s" % path) resp = self.speedy_conn.list_directory(path) if resp.status_code == 200: j = resp.json() return j["file-list"] elif resp.status_code == 404: raise exceptions.FileNotFoundError("no such directory: %s" % path) else: raise exceptions.UnspecifiedError("unexcept status code: %d" % path)
def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name)
def list_directory(self, path=None): try: path = self._init_path(path) if path and not path.endswith('/'): path += '/' files_generator = self._bucket.listdir(prefix=path) if len(list(files_generator)) == 0: raise Exception('empty') files_generator = self._bucket.listdir(prefix=path) for item in files_generator: yield item[0] except Exception: raise exceptions.FileNotFoundError('%s is not there' % path)
def get_size(self, path): path = self.getfullpath(path) logger.debug("get_size of %s", path) headers = {} r = self._oss.head_object(self.osscfg.bucket, path, headers) if (r.status / 100) != 2: raise exceptions.FileNotFoundError("Unable to get size of %s" % path) header_map = convert_header2map(r.getheaders()) size = safe_get_element("content-length", header_map) logger.debug("size of %s = %d", path, size) return size
def get_store(self, path, chunk_size=None): try: response = self._bucket[path] except KeyNotFound: raise exceptions.FileNotFoundError('%s is not there' % path) try: while True: chunk = response.read(chunk_size) if not chunk: break yield chunk except: raise IOError("Could not get content: %s" % path)
def stream_read(self, path, bytes_range=None): """Method to stream read.""" if not self.exists(path): raise exceptions.FileNotFoundError("File not found %s" % path) path = self._init_path(path) res = self.mos.get_object(self.bucket, path) if res.status == 200: block = res.read(self.buffer_size) while len(block) > 0: yield block block = res.read(self.buffer_size) else: raise IOError('read %s failed, status: %s' % (path, res.status))
def get_size(self, path): """Method to get the size.""" path = self._init_path(path) logging.debug("Get file size of %s" % path) try: file_size = self.mos.get_object_filesize(self.bucket, path) if file_size is not None: logging.debug("Size of %s is %s" % (path, file_size)) return file_size else: raise exceptions.FileNotFoundError("Unable to get size of %s" % path) except Exception: raise(exceptions.ConnectionError("Unable to get size of %s" % path))
def remove(self, path): logger.debug("remove path: %s" % path) resp = self.speedy_conn.delete(path) if resp.status_code == 204: logger.debug("speedy remove success: %s" % path) elif resp.status_code == 404: logger.warning("speedy remove, file not found: %s" % path) raise exceptions.FileNotFoundError("%s is not here" % path) else: logger.error("speedy remove, unexcept status code %d" % resp.status_code) raise exceptions.UnspecifiedError( "speedy remove, unexcept status code %d" % resp.status_code)
def stream_read(self, path, bytes_range=None): path = self.getfullpath(path) logger.debug("read from %s", path) if not self.exists(path): raise exceptions.FileNotFoundError( 'No such directory: \'{0}\''.format(path)) res = self._oss.get_object(self.osscfg.bucket, path) if res.status == 200: block = res.read(self.buffer_size) while len(block) > 0: yield block block = res.read(self.buffer_size) else: raise IOError('read %s failed, status: %s' % (path, res.status))
def remove(self, path): path = self._init_path(path) is_dir = False for item in self._bucket.listdir(prefix=path + '/'): self._bucket.delete(item[0]) is_dir = True if not is_dir: try: self._bucket.info(path) except Exception: raise exceptions.FileNotFoundError('%s is not there' % path) self._bucket.delete(path)
def list_directory(self, path=None): try: path = self._init_path(path) if path and not path.endswith('/'): path += '/' _, directory = self._swift_connection.get_container( container=self._swift_container, path=path) if not directory: raise for inode in directory: # trim extra trailing slashes if inode['name'].endswith('/'): inode['name'] = inode['name'][:-1] yield inode['name'].replace(self._root_path[1:] + '/', '', 1) except Exception: raise exceptions.FileNotFoundError('%s is not there' % path)
def s_remove(self, key): fail = False r = self._session.remove(key) r.wait() err = r.error() if err.code != 0: logger.warning("Unable to remove key %s %s", key, err.message) fail = True r = self._session.set_indexes(key, [], []) r.wait() err = r.error() if err.code != 0: logger.warning("Unable to remove key %s indexes %s", key, err.message) if fail: raise exceptions.FileNotFoundError("No such file %s" % key)
def _lsdir(self, path): """list all content in the target dir returns a list of filenames :param path: dir path """ dir_path = remove_slash(path or "") logger.info("list dir for {0}".format(dir_path)) res = self._request(GET, dir_path) for key, val in self.index_header.items(): if res.headers.get(key) != val: raise de.FileNotFoundError("{0} is not there".format(dir_path)) return [ os.path.join(dir_path, fname) for fname in res.text.splitlines() if fname ]