def transfer_image(self, lock): # NOTE(mikal): it is assumed the caller holds a lock on the artifact, and passes # it in lock. url, checksum, checksum_type = image_resolver.resolve(self.url) # If this is a request for a URL, do we have the most recent version # somewhere in the cluster? if not url.startswith(BLOB_URL): most_recent = self.__artifact.most_recent_index dirty = False if most_recent.get('index', 0) == 0: self.log.info('Cluster does not have a copy of image') dirty = True else: most_recent_blob = Blob.from_db(most_recent['blob_uuid']) resp = self._open_connection(url) if not most_recent_blob.modified: dirty = True elif most_recent_blob.modified != resp.headers.get( 'Last-Modified'): self.__artifact.add_event( 'image requires fetch', None, None, 'Last-Modified: %s -> %s' % (most_recent_blob.modified, resp.headers.get('Last-Modified'))) dirty = True if not most_recent_blob.size: dirty = True elif most_recent_blob.size != resp.headers.get( 'Content-Length'): self.__artifact.add_event( 'image requires fetch', None, None, 'Content-Length: %s -> %s' % (most_recent_blob.size, resp.headers.get('Content-Length'))) dirty = True if dirty: self.log.info('Cluster cached image is stale') else: url = '%s%s' % (BLOB_URL, most_recent_blob.uuid) self.log.info('Using cached image from cluster') # Ensure that we have the blob in the local store. This blob is in the # "original format" if downloaded from an HTTP source. if url.startswith(BLOB_URL): self.log.info('Fetching image from within the cluster') b = self._blob_get(lock, url) else: self.log.info('Fetching image from the internet') b = self._http_get_inner(lock, url, checksum, checksum_type) # Ref count increased here since it is known here whether the blob # will be used from within the cluster or newly created. b.ref_count_inc() return b
def get(self, blob_uuid=None, offset=0): # Ensure the blob exists b = Blob.from_db(blob_uuid) if not b: return api_base.error(404, 'blob not found') # Fast path if we have the blob locally os.makedirs(os.path.join(config.STORAGE_PATH, 'blobs'), exist_ok=True) blob_path = os.path.join(config.STORAGE_PATH, 'blobs', blob_uuid) if os.path.exists(blob_path): return flask.Response(flask.stream_with_context( _read_file(blob_path, offset)), mimetype='text/plain', status=200) # Otherwise find a node which has the blob and proxy. locations = b.locations if not locations: return api_base.error(404, 'blob missing') random.shuffle(locations) return flask.Response(flask.stream_with_context( _read_remote(locations[0], blob_uuid, offset=offset)), mimetype='text/plain', status=200)
def post(self, artifact_name=None, upload_uuid=None, source_url=None): u = Upload.from_db(upload_uuid) if not u: return api_base.error(404, 'upload not found') if u.node != config.NODE_NAME: url = 'http://%s:%d%s' % (u.node, config.API_PORT, flask.request.environ['PATH_INFO']) api_token = util_general.get_api_token( 'http://%s:%d' % (u.node, config.API_PORT), namespace=get_jwt_identity()[0]) r = requests.request( flask.request.environ['REQUEST_METHOD'], url, data=json.dumps(api_base.flask_get_post_body()), headers={ 'Authorization': api_token, 'User-Agent': util_general.get_user_agent() }) LOG.info('Proxied %s %s returns: %d, %s' % (flask.request.environ['REQUEST_METHOD'], url, r.status_code, r.text)) resp = flask.Response(r.text, mimetype='application/json') resp.status_code = r.status_code return resp if not source_url: source_url = ('%s%s/%s' % (UPLOAD_URL, get_jwt_identity()[0], artifact_name)) a = Artifact.from_url(Artifact.TYPE_IMAGE, source_url) with a.get_lock(ttl=(12 * constants.LOCK_REFRESH_SECONDS), timeout=config.MAX_IMAGE_TRANSFER_SECONDS): blob_uuid = str(uuid.uuid4()) blob_dir = os.path.join(config.STORAGE_PATH, 'blobs') blob_path = os.path.join(blob_dir, blob_uuid) upload_dir = os.path.join(config.STORAGE_PATH, 'uploads') upload_path = os.path.join(upload_dir, u.uuid) # NOTE(mikal): we can't use os.rename() here because these paths # might be on different filesystems. shutil.move(upload_path, blob_path) st = os.stat(blob_path) b = Blob.new( blob_uuid, st.st_size, time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime()), time.time()) b.state = Blob.STATE_CREATED b.ref_count_inc() b.observe() b.request_replication() a.state = Artifact.STATE_CREATED a.add_event('upload', None, None, 'success') a.add_index(b.uuid) a.state = Artifact.STATE_CREATED return a.external_view()
def get(self, artifact_uuid=None, artifact_from_db=None): retval = [] for idx in artifact_from_db.get_all_indexes(): b = Blob.from_db(idx['blob_uuid']) bout = b.external_view() bout['index'] = idx['index'] retval.append(bout) return retval
def _blob_get(self, lock, url): """Fetch a blob from the cluster.""" blob_uuid = url[len(BLOB_URL):] b = Blob.from_db(blob_uuid) if not b: raise exceptions.BlobMissing(blob_uuid) b.ensure_local([lock]) return b
def get(self, node=None): retval = [] with etcd.ThreadLocalReadOnlyCache(): for a in Artifacts(filters=[baseobject.active_states_filter]): if node: idx = a.most_recent_index if 'blob_uuid' in idx: b = Blob.from_db(idx['blob_uuid']) if b and node in b.locations: retval.append(a.external_view()) else: retval.append(a.external_view()) return retval
def post(self, label_name=None, blob_uuid=None, max_versions=0): b = Blob.from_db(blob_uuid) if not b: return api_base.error(404, 'blob not found') try: b.ref_count_inc() except BlobDeleted: return api_base.error(400, 'blob has been deleted') a = Artifact.from_url(Artifact.TYPE_LABEL, _label_url(label_name), max_versions) a.add_index(blob_uuid) a.state = dbo.STATE_CREATED return a.external_view()
def delete(self, label_name=None): artifacts = list( Artifacts(filters=[ partial(type_filter, Artifact.TYPE_LABEL), partial(url_filter, _label_url(label_name)), active_states_filter ])) if len(artifacts) == 0: api_base.error(404, 'label %s not found' % label_name) for a in artifacts: a.state = dbo.STATE_DELETED for blob_index in a.get_all_indexes(): b = Blob.from_db(blob_index['blob_uuid']) b.ref_count_dec()
def delete(self, artifact_uuid=None, artifact_from_db=None): """Delete an artifact from the cluster Artifacts can only be deleted from the system if they are not in use. The actual deletion of the on-disk files is left to the cleaner daemon. It is acknowledged that there is a potential race condition between the check that an artifact is not in use and the marking of the artifact as deleted. This is only caused by a user simultaneously deleting an artifact and attempting to start a VM using it. It is recommended that the user does not do that. """ # TODO(andy): Enforce namespace permissions when snapshots have namespaces # TODO(mikal): this should all be refactored to be in the object if artifact_from_db.state.value == Artifact.STATE_DELETED: # Already deleted, nothing to do. return # Check for instances using a blob referenced by the artifact. blobs = [] sole_ref_in_use = [] for blob_index in artifact_from_db.get_all_indexes(): b = Blob.from_db(blob_index['blob_uuid']) if b: blobs.append(b) if b.ref_count == 1: sole_ref_in_use += b.instances if sole_ref_in_use: return api_base.error( 400, 'Cannot delete last reference to blob in use by instance (%s)' % (', '.join(sole_ref_in_use), )) artifact_from_db.delete() for b in blobs: b.ref_count_dec()
def _maintain_blobs(self): # Find orphaned and deleted blobs still on disk blob_path = os.path.join(config.STORAGE_PATH, 'blobs') os.makedirs(blob_path, exist_ok=True) cache_path = os.path.join(config.STORAGE_PATH, 'image_cache') os.makedirs(cache_path, exist_ok=True) for ent in os.listdir(blob_path): entpath = os.path.join(blob_path, ent) st = os.stat(entpath) # If we've had this file for more than two cleaner delays... if time.time() - st.st_mtime > config.CLEANER_DELAY * 2: if ent.endswith('.partial'): # ... and its a stale partial transfer LOG.with_fields({ 'blob': ent }).warning('Deleting stale partial transfer') os.unlink(entpath) else: b = Blob.from_db(ent) if (not b or b.state.value == Blob.STATE_DELETED or config.NODE_NAME not in b.locations): LOG.with_fields({ 'blob': ent }).warning('Deleting orphaned blob') os.unlink(entpath) cached = util_general.file_permutation_exists( os.path.join(cache_path, ent), ['iso', 'qcow2']) if cached: os.unlink(cached) # Find transcoded blobs in the image cache which are no longer in use for ent in os.listdir(cache_path): entpath = os.path.join(cache_path, ent) # Broken symlinks will report an error here that we have to catch try: st = os.stat(entpath) except OSError as e: if e.errno == errno.ENOENT: LOG.with_fields({ 'blob': ent }).warning('Deleting broken symlinked image cache entry') os.unlink(entpath) continue else: raise e # If we haven't seen this file in use for more than two cleaner delays... if time.time() - st.st_mtime > config.CLEANER_DELAY * 2: blob_uuid = ent.split('.')[0] b = Blob.from_db(blob_uuid) if not b: LOG.with_fields({ 'blob': ent }).warning('Deleting orphaned image cache entry') os.unlink(entpath) continue if b.ref_count == 0: LOG.with_fields({ 'blob': ent }).warning('Deleting globally unused image cache entry') os.unlink(entpath) continue this_node = 0 for instance_uuid in b.instances: i = instance.Instance.from_db(instance_uuid) if i: if i.placement.get('node') == config.NODE_NAME: this_node += 1 LOG.with_fields({ 'blob': blob_uuid, 'this_node': this_node }).info('Blob users on this node') if this_node == 0: LOG.with_fields({ 'blob': blob_uuid }).warning('Deleting unused image cache entry') os.unlink(entpath) else: # Record that this file is in use for the benefit of # the above time check. pathlib.Path(entpath).touch(exist_ok=True) # Find blobs which should be on this node but are not. missing = [] with etcd.ThreadLocalReadOnlyCache(): for b in Blobs([active_states_filter]): if config.NODE_NAME in b.locations: if not os.path.exists( os.path.join(config.STORAGE_PATH, 'blobs', b.uuid)): missing.append(b.uuid) for blob_uuid in missing: b = Blob.from_db(blob_uuid) if b: LOG.with_fields({ 'blob': blob_uuid }).warning('Blob missing from node') b.drop_node_location(config.NODE_NAME)