def wrapper(*args, **kwargs):
i = kwargs.get('instance_from_db')
if not i:
return
placement = i.placement
if not placement:
return
if not placement.get('node'):
return
if placement.get('node') != config.NODE_NAME:
url = 'http://%s:%d%s' % (placement['node'], config.API_PORT,
flask.request.environ['PATH_INFO'])
api_token = util_general.get_api_token(
'http://%s:%d' % (placement['node'], config.API_PORT),
namespace=get_jwt_identity()[0])
r = requests.request(
flask.request.environ['REQUEST_METHOD'], url,
data=json.dumps(flask_get_post_body()),
headers={'Authorization': api_token,
'User-Agent': util_general.get_user_agent()})
LOG.info('Proxied %s %s returns: %d, %s' % (
flask.request.environ['REQUEST_METHOD'], url,
r.status_code, r.text))
resp = flask.Response(r.text,
mimetype='application/json')
resp.status_code = r.status_code
return resp
return func(*args, **kwargs)
def wrapper(*args, **kwargs):
u = kwargs.get('upload_from_db')
if not u:
return
if not u.node:
return
if u.node != config.NODE_NAME:
url = 'http://%s:%d%s' % (u.node, config.API_PORT,
flask.request.environ['PATH_INFO'])
api_token = util_general.get_api_token(
'http://%s:%d' % (u.node, config.API_PORT),
namespace=get_jwt_identity()[0])
r = requests.request(
flask.request.environ['REQUEST_METHOD'], url,
data=flask.request.get_data(cache=False, as_text=False,
parse_form_data=False),
headers={'Authorization': api_token,
'User-Agent': util_general.get_user_agent()})
LOG.info('Proxied %s %s returns: %d, %s' % (
flask.request.environ['REQUEST_METHOD'], url,
r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
return func(*args, **kwargs)
def post(self, artifact_name=None, upload_uuid=None, source_url=None):
u = Upload.from_db(upload_uuid)
if not u:
return api_base.error(404, 'upload not found')
if u.node != config.NODE_NAME:
url = 'http://%s:%d%s' % (u.node, config.API_PORT,
flask.request.environ['PATH_INFO'])
api_token = util_general.get_api_token(
'http://%s:%d' % (u.node, config.API_PORT),
namespace=get_jwt_identity()[0])
r = requests.request(
flask.request.environ['REQUEST_METHOD'],
url,
data=json.dumps(api_base.flask_get_post_body()),
headers={
'Authorization': api_token,
'User-Agent': util_general.get_user_agent()
})
LOG.info('Proxied %s %s returns: %d, %s' %
(flask.request.environ['REQUEST_METHOD'], url,
r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
if not source_url:
source_url = ('%s%s/%s' %
(UPLOAD_URL, get_jwt_identity()[0], artifact_name))
a = Artifact.from_url(Artifact.TYPE_IMAGE, source_url)
with a.get_lock(ttl=(12 * constants.LOCK_REFRESH_SECONDS),
timeout=config.MAX_IMAGE_TRANSFER_SECONDS):
blob_uuid = str(uuid.uuid4())
blob_dir = os.path.join(config.STORAGE_PATH, 'blobs')
blob_path = os.path.join(blob_dir, blob_uuid)
upload_dir = os.path.join(config.STORAGE_PATH, 'uploads')
upload_path = os.path.join(upload_dir, u.uuid)
# NOTE(mikal): we can't use os.rename() here because these paths
# might be on different filesystems.
shutil.move(upload_path, blob_path)
st = os.stat(blob_path)
b = Blob.new(
blob_uuid, st.st_size,
time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime()),
time.time())
b.state = Blob.STATE_CREATED
b.ref_count_inc()
b.observe()
b.request_replication()
a.state = Artifact.STATE_CREATED
a.add_event('upload', None, None, 'success')
a.add_index(b.uuid)
a.state = Artifact.STATE_CREATED
return a.external_view()
def _read_remote(target, blob_uuid, offset=0):
api_token = util_general.get_api_token('http://%s:%d' %
(target, config.API_PORT),
namespace=get_jwt_identity()[0])
url = 'http://%s:%d/blob/%s?offset=%d' % (target, config.API_PORT,
blob_uuid, offset)
r = requests.request('GET',
url,
stream=True,
headers={
'Authorization': api_token,
'User-Agent': util_general.get_user_agent()
})
for chunk in r.iter_content(chunk_size=8192):
yield chunk
def wrapper(*args, **kwargs):
if not config.NODE_IS_NETWORK_NODE:
admin_token = util_general.get_api_token(
'http://%s:%d' % (config.NETWORK_NODE_IP, config.API_PORT),
namespace='system')
r = requests.request(
flask.request.environ['REQUEST_METHOD'],
'http://%s:%d%s'
% (config.NETWORK_NODE_IP, config.API_PORT,
flask.request.environ['PATH_INFO']),
data=flask.request.data,
headers={'Authorization': admin_token,
'User-Agent': util_general.get_user_agent()})
LOG.info('Returning proxied request: %d, %s'
% (r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
return func(*args, **kwargs)
def ensure_local(self, locks):
with self.get_lock(config.NODE_NAME) as blob_lock:
if self.state.value != self.STATE_CREATED:
self.log.warning(
'Blob not in created state, replication cancelled')
return
blob_dir = os.path.join(config.STORAGE_PATH, 'blobs')
blob_path = os.path.join(blob_dir, self.uuid)
os.makedirs(blob_dir, exist_ok=True)
if os.path.exists(blob_path):
self.log.info('Blob already exists!')
self.observe()
return
locations = self.locations
random.shuffle(locations)
blob_source = locations[0]
with open(blob_path + '.partial', 'wb') as f:
done = False
last_refresh = 0
refreshable_locks = locks.copy()
refreshable_locks.append(blob_lock)
total_bytes_received = 0
previous_percentage = 0
while not done:
bytes_in_attempt = 0
try:
admin_token = util_general.get_api_token(
'http://%s:%d' % (blob_source, config.API_PORT))
url = ('http://%s:%d/blob/%s?offset=%d' %
(blob_source, config.API_PORT, self.uuid,
total_bytes_received))
r = requests.request('GET',
url,
stream=True,
headers={
'Authorization':
admin_token,
'User-Agent':
util_general.get_user_agent()
})
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
total_bytes_received += len(chunk)
bytes_in_attempt += len(chunk)
if time.time(
) - last_refresh > LOCK_REFRESH_SECONDS:
db.refresh_locks(refreshable_locks)
last_refresh = time.time()
percentage = (total_bytes_received /
int(self.size) * 100.0)
if (percentage - previous_percentage) > 10.0:
self.log.with_fields({
'bytes_fetched': total_bytes_received,
'size': int(self.size)
}).info('Fetch %.02f percent complete' %
percentage)
previous_percentage = percentage
done = True
self.log.with_fields({
'bytes_fetched': total_bytes_received,
'size': int(self.size),
'done': done
}).info('HTTP request ran out of chunks')
except (http.client.IncompleteRead,
urllib3.exceptions.ProtocolError,
requests.exceptions.ChunkedEncodingError) as e:
# An API error (or timeout) occured. Retry unless we got nothing.
if bytes_in_attempt > 0:
self.log.info('HTTP connection dropped, retrying')
else:
self.log.error('HTTP connection dropped without '
'transferring data: %s' % e)
raise e
if total_bytes_received != int(self.size):
if os.path.exists(blob_path + '.partial'):
os.unlink(blob_path + '.partial')
raise BlobFetchFailed('Did not fetch enough data')
self.log.info('Completing transfer')
os.rename(blob_path + '.partial', blob_path)
self.log.with_fields({
'bytes_fetched': total_bytes_received,
'size': int(self.size)
}).info('Fetch complete')
self.observe()
return total_bytes_received