def put_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
librato.count('blueprint-io-server.bandwidth.in', request.content_length)
statsd.update('blueprint-io-server.bandwidth.in', request.content_length)
validate_content_length()
# Validate the tarball content.
if hashlib.sha1(request.data).hexdigest() != sha:
abort(400)
# Ensure the tarball appears in the blueprint.
b = _blueprint(secret, name)
if b is None:
abort(404)
elif b is False:
abort(502)
if '{0}.tar'.format(sha) not in b.sources.itervalues():
abort(400)
# Store the tarball in S3.
if not backend.put_tarball(secret, name, sha, request.data):
abort(502)
return MeteredResponse(response='',
status=202,
content_type='text/plain')
def put_blueprint(secret, name):
validate_secret(secret)
validate_name(name)
librato.count('blueprint-io-server.bandwidth.in', request.content_length)
statsd.update('blueprint-io-server.bandwidth.in', request.content_length)
validate_content_length()
# Validate the blueprint JSON format. This could stand more rigor
# or, dare I say it, a schema?
try:
for k in request.json.iterkeys():
if k not in ('arch', 'files', 'packages', 'services', 'sources'):
abort(400)
except ValueError:
abort(400)
# Remove tarballs referenced by the old blueprint but not the new one.
b = _blueprint(secret, name)
if b is not None and b is not False:
for filename in set(b.sources.itervalues()) - \
set(request.json.get('sources', {}).itervalues()):
backend.delete_tarball(secret, name, filename[0:-4])
# Store the blueprint JSON in S3.
if not backend.put_blueprint(secret, name, request.data):
abort(502)
return MeteredResponse(response='',
status=202,
content_type='text/plain')
def put_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
librato.count('blueprint-io-server.bandwidth.in', request.content_length)
statsd.update('blueprint-io-server.bandwidth.in', request.content_length)
validate_content_length()
# Validate the tarball content.
if hashlib.sha1(request.data).hexdigest() != sha:
abort(400)
# Ensure the tarball appears in the blueprint.
b = _blueprint(secret, name)
if b is None:
abort(404)
elif b is False:
abort(502)
if '{0}.tar'.format(sha) not in b.sources.itervalues():
abort(400)
# Store the tarball in S3.
if not backend.put_tarball(secret, name, sha, request.data):
abort(502)
return MeteredResponse(response='', status=202, content_type='text/plain')
def put_blueprint(secret, name):
validate_secret(secret)
validate_name(name)
librato.count('blueprint-io-server.bandwidth.in', request.content_length)
statsd.update('blueprint-io-server.bandwidth.in', request.content_length)
validate_content_length()
# Validate the blueprint JSON format. This could stand more rigor
# or, dare I say it, a schema?
try:
for k in request.json.iterkeys():
if k not in ('arch', 'files', 'packages', 'services', 'sources'):
abort(400)
except ValueError:
abort(400)
# Remove tarballs referenced by the old blueprint but not the new one.
b = _blueprint(secret, name)
if b is not None and b is not False:
for filename in set(b.sources.itervalues()) - \
set(request.json.get('sources', {}).itervalues()):
backend.delete_tarball(secret, name, filename[0:-4])
# Store the blueprint JSON in S3.
if not backend.put_blueprint(secret, name, request.data):
abort(502)
return MeteredResponse(response='', status=202, content_type='text/plain')
def list(key):
librato.count('blueprint-acmeaws-com-server.requests.list')
statsd.increment('blueprint-acmeaws-com-server.requests.list')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
return b.list(key)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def list(key):
"""
List objects in MongoDB whose key begins with the given prefix
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
try:
result = collection.find({"key" : '^%s' % (key)})
except:
return False
return result
def delete(key):
"""
Remove an object from MongoDB.
"""
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-io-server.requests.delete')
statsd.increment('blueprint-io-server.requests.delete')
try:
collection.delete({"key" : key})
except:
return False
def get(key):
"""
Fetch an object from MongoDB.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
try:
k = collection.find_one({"key" : key})
if k is None:
return None
except:
return False
return k['tarball']
def list(key):
librato.count('blueprint-acmeaws-com-server.requests.list')
statsd.increment('blueprint-acmeaws-com-server.requests.list')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
return b.list(key)
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def head(key):
librato.count('blueprint-acmeaws-com-server.requests.head')
statsd.increment('blueprint-acmeaws-com-server.requests.head')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def put(key, data):
"""
Store an object in MongoDB.
"""
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
# TODO librato.something('blueprint-io-server.storage', len(data))
statsd.update('blueprint-io-server.storage', len(data))
element = StoredObject(key, data)
try:
collection.insert(element.__dict__)
except:
return False
return True
def head(key):
"""
Make a HEAD request for an object in MongoDB.
This returns the size of the tarball.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
try:
k = collection.find_one({"key" : key})
if k is None:
return None
except:
return False
return len(k['tarball'])
def delete(key):
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-acmeaws-com-server.requests.delete')
statsd.increment('blueprint-acmeaws-com-server.requests.delete')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
statsd.update('blueprint-acmeaws-com-server.storage', -content_length)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def put(key, data):
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def list(key):
"""
List objects in S3 whose keys begin with the given prefix. This
function makes at least one billable request.
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
return b.list(key)
try:
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def get_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
content_length = backend.head_tarball(secret, name, sha)
if content_length is None:
abort(404)
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_tarball(secret, name, sha), code=301)
def get(key):
"""
Fetch an object from S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
return k.get_contents_as_string()
except boto.exception.S3ResponseError:
return None
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def head(key):
librato.count('blueprint-acmeaws-com-server.requests.head')
statsd.increment('blueprint-acmeaws-com-server.requests.head')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def get_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
content_length = backend.head_tarball(secret, name, sha)
if content_length is None:
abort(404)
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_tarball(secret, name, sha), code=301)
def delete(key):
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-acmeaws-com-server.requests.delete')
statsd.increment('blueprint-acmeaws-com-server.requests.delete')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
statsd.update('blueprint-acmeaws-com-server.storage', -content_length)
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def put(key, data):
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def list(key):
"""
List objects in S3 whose keys begin with the given prefix. This
function makes at least one billable request.
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
return b.list(key)
try:
return True
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def head(key):
"""
Make a HEAD request for an object in S3. This is needed to find the
object's length so it can be accounted. This function makes one
billable request and anticipates another.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def get(key):
"""
Fetch an object from S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
return k.get_contents_as_string()
except boto.exception.S3ResponseError:
return None
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def put(key, data):
"""
Store an object in S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
# TODO librato.something('blueprint-io-server.storage', len(data))
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def delete(key):
"""
Remove an object from S3. DELETE requests are free but this function
still makes one billable request to account for freed storage.
"""
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-io-server.requests.delete')
statsd.increment('blueprint-io-server.requests.delete')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
# TODO librato.something('blueprint-io-server.storage', -content_length)
statsd.update('blueprint-io-server.storage', -content_length)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
boto.exception.S3ResponseError, httplib.HTTPException,
socket.error, socket.gaierror):
return False
def head(key):
"""
Make a HEAD request for an object in S3. This is needed to find the
object's length so it can be accounted. This function makes one
billable request and anticipates another.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def put(key, data):
"""
Store an object in S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
# TODO librato.something('blueprint-io-server.storage', len(data))
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def delete(key):
"""
Remove an object from S3. DELETE requests are free but this function
still makes one billable request to account for freed storage.
"""
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-io-server.requests.delete')
statsd.increment('blueprint-io-server.requests.delete')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
# TODO librato.something('blueprint-io-server.storage', -content_length)
statsd.update('blueprint-io-server.storage', -content_length)
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
boto.exception.S3ResponseError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def get_blueprint(secret, name):
validate_secret(secret)
validate_name(name)
content_length = backend.head_blueprint(secret, name)
if content_length is None:
abort(404)
# Pretty HTML for browsers.
if browser_pattern.search(request.environ.get('HTTP_USER_AGENT', '')) \
or 'html' == request.args.get('format'):
librato.count('blueprint-io-server.renders')
statsd.increment('blueprint-io-server.renders')
return render_template('blueprint.html', b=_blueprint(secret, name))
# Raw JSON for everybody else.
else:
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_blueprint(secret, name), code=301)
def get_blueprint(secret, name):
validate_secret(secret)
validate_name(name)
content_length = backend.head_blueprint(secret, name)
if content_length is None:
abort(404)
# Pretty HTML for browsers.
if browser_pattern.search(request.environ.get('HTTP_USER_AGENT', '')) \
or 'html' == request.args.get('format'):
librato.count('blueprint-io-server.renders')
statsd.increment('blueprint-io-server.renders')
return render_template('blueprint.html', b=_blueprint(secret, name))
# Raw JSON for everybody else.
else:
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_blueprint(secret, name), code=301)
def __init__(self, *args, **kwargs):
super(MeteredResponse, self).__init__(*args, **kwargs)
content_length = len(kwargs.get('response', ''))
if 0 < content_length:
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
def __init__(self, *args, **kwargs):
super(MeteredResponse, self).__init__(*args, **kwargs)
content_length = len(kwargs.get('response', ''))
if 0 < content_length:
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)