def transfer_image(self, lock):
# NOTE(mikal): it is assumed the caller holds a lock on the artifact, and passes
# it in lock.
url, checksum, checksum_type = image_resolver.resolve(self.url)
# If this is a request for a URL, do we have the most recent version
# somewhere in the cluster?
if not url.startswith(BLOB_URL):
most_recent = self.__artifact.most_recent_index
dirty = False
if most_recent.get('index', 0) == 0:
self.log.info('Cluster does not have a copy of image')
dirty = True
else:
most_recent_blob = Blob.from_db(most_recent['blob_uuid'])
resp = self._open_connection(url)
if not most_recent_blob.modified:
dirty = True
elif most_recent_blob.modified != resp.headers.get(
'Last-Modified'):
self.__artifact.add_event(
'image requires fetch', None, None,
'Last-Modified: %s -> %s' %
(most_recent_blob.modified,
resp.headers.get('Last-Modified')))
dirty = True
if not most_recent_blob.size:
dirty = True
elif most_recent_blob.size != resp.headers.get(
'Content-Length'):
self.__artifact.add_event(
'image requires fetch', None, None,
'Content-Length: %s -> %s' %
(most_recent_blob.size,
resp.headers.get('Content-Length')))
dirty = True
if dirty:
self.log.info('Cluster cached image is stale')
else:
url = '%s%s' % (BLOB_URL, most_recent_blob.uuid)
self.log.info('Using cached image from cluster')
# Ensure that we have the blob in the local store. This blob is in the
# "original format" if downloaded from an HTTP source.
if url.startswith(BLOB_URL):
self.log.info('Fetching image from within the cluster')
b = self._blob_get(lock, url)
else:
self.log.info('Fetching image from the internet')
b = self._http_get_inner(lock, url, checksum, checksum_type)
# Ref count increased here since it is known here whether the blob
# will be used from within the cluster or newly created.
b.ref_count_inc()
return b
def get(self, blob_uuid=None, offset=0):
# Ensure the blob exists
b = Blob.from_db(blob_uuid)
if not b:
return api_base.error(404, 'blob not found')
# Fast path if we have the blob locally
os.makedirs(os.path.join(config.STORAGE_PATH, 'blobs'), exist_ok=True)
blob_path = os.path.join(config.STORAGE_PATH, 'blobs', blob_uuid)
if os.path.exists(blob_path):
return flask.Response(flask.stream_with_context(
_read_file(blob_path, offset)),
mimetype='text/plain',
status=200)
# Otherwise find a node which has the blob and proxy.
locations = b.locations
if not locations:
return api_base.error(404, 'blob missing')
random.shuffle(locations)
return flask.Response(flask.stream_with_context(
_read_remote(locations[0], blob_uuid, offset=offset)),
mimetype='text/plain',
status=200)
def post(self, artifact_name=None, upload_uuid=None, source_url=None):
u = Upload.from_db(upload_uuid)
if not u:
return api_base.error(404, 'upload not found')
if u.node != config.NODE_NAME:
url = 'http://%s:%d%s' % (u.node, config.API_PORT,
flask.request.environ['PATH_INFO'])
api_token = util_general.get_api_token(
'http://%s:%d' % (u.node, config.API_PORT),
namespace=get_jwt_identity()[0])
r = requests.request(
flask.request.environ['REQUEST_METHOD'],
url,
data=json.dumps(api_base.flask_get_post_body()),
headers={
'Authorization': api_token,
'User-Agent': util_general.get_user_agent()
})
LOG.info('Proxied %s %s returns: %d, %s' %
(flask.request.environ['REQUEST_METHOD'], url,
r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
if not source_url:
source_url = ('%s%s/%s' %
(UPLOAD_URL, get_jwt_identity()[0], artifact_name))
a = Artifact.from_url(Artifact.TYPE_IMAGE, source_url)
with a.get_lock(ttl=(12 * constants.LOCK_REFRESH_SECONDS),
timeout=config.MAX_IMAGE_TRANSFER_SECONDS):
blob_uuid = str(uuid.uuid4())
blob_dir = os.path.join(config.STORAGE_PATH, 'blobs')
blob_path = os.path.join(blob_dir, blob_uuid)
upload_dir = os.path.join(config.STORAGE_PATH, 'uploads')
upload_path = os.path.join(upload_dir, u.uuid)
# NOTE(mikal): we can't use os.rename() here because these paths
# might be on different filesystems.
shutil.move(upload_path, blob_path)
st = os.stat(blob_path)
b = Blob.new(
blob_uuid, st.st_size,
time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime()),
time.time())
b.state = Blob.STATE_CREATED
b.ref_count_inc()
b.observe()
b.request_replication()
a.state = Artifact.STATE_CREATED
a.add_event('upload', None, None, 'success')
a.add_index(b.uuid)
a.state = Artifact.STATE_CREATED
return a.external_view()
def get(self, artifact_uuid=None, artifact_from_db=None):
retval = []
for idx in artifact_from_db.get_all_indexes():
b = Blob.from_db(idx['blob_uuid'])
bout = b.external_view()
bout['index'] = idx['index']
retval.append(bout)
return retval
def _blob_get(self, lock, url):
"""Fetch a blob from the cluster."""
blob_uuid = url[len(BLOB_URL):]
b = Blob.from_db(blob_uuid)
if not b:
raise exceptions.BlobMissing(blob_uuid)
b.ensure_local([lock])
return b
def get(self, node=None):
retval = []
with etcd.ThreadLocalReadOnlyCache():
for a in Artifacts(filters=[baseobject.active_states_filter]):
if node:
idx = a.most_recent_index
if 'blob_uuid' in idx:
b = Blob.from_db(idx['blob_uuid'])
if b and node in b.locations:
retval.append(a.external_view())
else:
retval.append(a.external_view())
return retval
def post(self, label_name=None, blob_uuid=None, max_versions=0):
b = Blob.from_db(blob_uuid)
if not b:
return api_base.error(404, 'blob not found')
try:
b.ref_count_inc()
except BlobDeleted:
return api_base.error(400, 'blob has been deleted')
a = Artifact.from_url(Artifact.TYPE_LABEL, _label_url(label_name),
max_versions)
a.add_index(blob_uuid)
a.state = dbo.STATE_CREATED
return a.external_view()
def delete(self, label_name=None):
artifacts = list(
Artifacts(filters=[
partial(type_filter, Artifact.TYPE_LABEL),
partial(url_filter, _label_url(label_name)),
active_states_filter
]))
if len(artifacts) == 0:
api_base.error(404, 'label %s not found' % label_name)
for a in artifacts:
a.state = dbo.STATE_DELETED
for blob_index in a.get_all_indexes():
b = Blob.from_db(blob_index['blob_uuid'])
b.ref_count_dec()
def delete(self, artifact_uuid=None, artifact_from_db=None):
"""Delete an artifact from the cluster
Artifacts can only be deleted from the system if they are not in use.
The actual deletion of the on-disk files is left to the cleaner daemon.
It is acknowledged that there is a potential race condition between the
check that an artifact is not in use and the marking of the artifact as
deleted. This is only caused by a user simultaneously deleting an
artifact and attempting to start a VM using it. It is recommended that
the user does not do that.
"""
# TODO(andy): Enforce namespace permissions when snapshots have namespaces
# TODO(mikal): this should all be refactored to be in the object
if artifact_from_db.state.value == Artifact.STATE_DELETED:
# Already deleted, nothing to do.
return
# Check for instances using a blob referenced by the artifact.
blobs = []
sole_ref_in_use = []
for blob_index in artifact_from_db.get_all_indexes():
b = Blob.from_db(blob_index['blob_uuid'])
if b:
blobs.append(b)
if b.ref_count == 1:
sole_ref_in_use += b.instances
if sole_ref_in_use:
return api_base.error(
400,
'Cannot delete last reference to blob in use by instance (%s)'
% (', '.join(sole_ref_in_use), ))
artifact_from_db.delete()
for b in blobs:
b.ref_count_dec()
def _maintain_blobs(self):
# Find orphaned and deleted blobs still on disk
blob_path = os.path.join(config.STORAGE_PATH, 'blobs')
os.makedirs(blob_path, exist_ok=True)
cache_path = os.path.join(config.STORAGE_PATH, 'image_cache')
os.makedirs(cache_path, exist_ok=True)
for ent in os.listdir(blob_path):
entpath = os.path.join(blob_path, ent)
st = os.stat(entpath)
# If we've had this file for more than two cleaner delays...
if time.time() - st.st_mtime > config.CLEANER_DELAY * 2:
if ent.endswith('.partial'):
# ... and its a stale partial transfer
LOG.with_fields({
'blob': ent
}).warning('Deleting stale partial transfer')
os.unlink(entpath)
else:
b = Blob.from_db(ent)
if (not b or b.state.value == Blob.STATE_DELETED
or config.NODE_NAME not in b.locations):
LOG.with_fields({
'blob': ent
}).warning('Deleting orphaned blob')
os.unlink(entpath)
cached = util_general.file_permutation_exists(
os.path.join(cache_path, ent), ['iso', 'qcow2'])
if cached:
os.unlink(cached)
# Find transcoded blobs in the image cache which are no longer in use
for ent in os.listdir(cache_path):
entpath = os.path.join(cache_path, ent)
# Broken symlinks will report an error here that we have to catch
try:
st = os.stat(entpath)
except OSError as e:
if e.errno == errno.ENOENT:
LOG.with_fields({
'blob': ent
}).warning('Deleting broken symlinked image cache entry')
os.unlink(entpath)
continue
else:
raise e
# If we haven't seen this file in use for more than two cleaner delays...
if time.time() - st.st_mtime > config.CLEANER_DELAY * 2:
blob_uuid = ent.split('.')[0]
b = Blob.from_db(blob_uuid)
if not b:
LOG.with_fields({
'blob': ent
}).warning('Deleting orphaned image cache entry')
os.unlink(entpath)
continue
if b.ref_count == 0:
LOG.with_fields({
'blob': ent
}).warning('Deleting globally unused image cache entry')
os.unlink(entpath)
continue
this_node = 0
for instance_uuid in b.instances:
i = instance.Instance.from_db(instance_uuid)
if i:
if i.placement.get('node') == config.NODE_NAME:
this_node += 1
LOG.with_fields({
'blob': blob_uuid,
'this_node': this_node
}).info('Blob users on this node')
if this_node == 0:
LOG.with_fields({
'blob': blob_uuid
}).warning('Deleting unused image cache entry')
os.unlink(entpath)
else:
# Record that this file is in use for the benefit of
# the above time check.
pathlib.Path(entpath).touch(exist_ok=True)
# Find blobs which should be on this node but are not.
missing = []
with etcd.ThreadLocalReadOnlyCache():
for b in Blobs([active_states_filter]):
if config.NODE_NAME in b.locations:
if not os.path.exists(
os.path.join(config.STORAGE_PATH, 'blobs',
b.uuid)):
missing.append(b.uuid)
for blob_uuid in missing:
b = Blob.from_db(blob_uuid)
if b:
LOG.with_fields({
'blob': blob_uuid
}).warning('Blob missing from node')
b.drop_node_location(config.NODE_NAME)