def configure_versioning(self, versioning):
"""
set the bucket's versioning property to True or False
"""
http_connection = HTTPConnection(
compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id
)
method = "PUT"
uri = compute_uri(
"/".join([
"customers",
self._identity.user_name,
"collections",
self._collection_name
]),
versioning=repr(versioning)
)
self._log.info("putting {0}".format(uri))
response = http_connection.request(method, uri)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
assert result["success"]
self._versioning = versioning
def get_space_used(self):
"""
get disk space statistics for this bucket
"""
http_connection = HTTPConnection(
compute_default_hostname(),
self._config.user_name,
self._config.auth_key,
self._config.auth_key_id
)
method = "GET"
uri = compute_uri(
"/".join([
"customers",
self._config.user_name,
"collections",
self._collection_name
]),
action="space_usage"
)
response = http_connection.request(method, uri)
data = response.read()
http_connection.close()
return json.loads(data)
def delete_bucket(self, bucket_name):
method = "DELETE"
http_connection = HTTPConnection(
compute_default_hostname(),
self._config.user_name,
self._config.auth_key,
self._config.auth_key_id
)
if bucket_name.startswith("/"):
bucket_name = bucket_name[1:]
uri = compute_uri(
"/".join([
"customers",
self._config.user_name,
"collections",
bucket_name
]),
)
self._log.info("requesting %s" % (uri, ))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError, instance:
self._log.error(str(instance))
http_connection.close()
raise
def delete_bucket(self, bucket_name):
"""
remove (an empty) bucket from nimbus.io
This operation will fail if the collection contains any active keys.
When tis operaton succeeds, the colection/bucket name is available for
re-use.
"""
method = "DELETE"
http_connection = HTTPConnection(compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id)
if bucket_name.startswith("/"):
bucket_name = bucket_name[1:]
uri = compute_uri(
"/".join([
"customers", self._identity.user_name, "collections",
bucket_name
]), )
self._log.info("requesting {0}".format(uri))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
response.read()
http_connection.close()
def _space_usage(args, identity, ncl_dict):
method = "GET"
if identity is None:
raise InvalidIdentity("Must have identity to retrieve space usage")
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name, identity.auth_key,
identity.auth_key_id)
kwargs = {"action": "space_usage"}
if "days" in ncl_dict:
kwargs["days_of_history"] = ncl_dict["days"]
path = "/".join([
"customers", identity.user_name, "collections",
ncl_dict["collection_name"]
])
uri = compute_uri(path, **kwargs)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
if not result["success"]:
raise NCLErrorResult(result["error_message"])
print()
for day_entry in result["operational_stats"]:
print(day_entry["day"])
if day_entry["archive_success"] != 0:
print("{0:8} archive success".format(day_entry["archive_success"]))
print("{0:8} archive bytes".format(day_entry["success_bytes_in"]))
if day_entry["retrieve_success"] != 0:
print("{0:8} retrieve success".format(
day_entry["retrieve_success"]))
print("{0:8} retrieve bytes".format(
day_entry["success_bytes_out"]))
if day_entry["delete_success"] != 0:
print("{0:8} delete success".format(day_entry["delete_success"]))
if day_entry["listmatch_success"] != 0:
print("{0:8} listmatch success".format(
day_entry["listmatch_success"]))
def _list_collection(args, identity, ncl_dict):
method = "GET"
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name, identity.auth_key,
identity.auth_key_id)
path = "/".join([
"customers", identity.user_name, "collections",
ncl_dict["collection_name"]
])
uri = compute_uri(path)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
print(str(result))
def _list_collection(args, identity, ncl_dict):
method = "GET"
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name,
identity.auth_key,
identity.auth_key_id)
path = "/".join(["customers",
identity.user_name,
"collections",
ncl_dict["collection_name"]])
uri = compute_uri(path)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
print(str(result))
def _space_usage(args, identity, ncl_dict):
method = "GET"
if identity is None:
raise InvalidIdentity("Must have identity to retrieve space usage")
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name,
identity.auth_key,
identity.auth_key_id)
kwargs = {"action" : "space_usage"}
if "days" in ncl_dict:
kwargs["days_of_history"] = ncl_dict["days"]
path = "/".join(["customers",
identity.user_name,
"collections",
ncl_dict["collection_name"]])
uri = compute_uri(path, **kwargs)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
if not result["success"]:
raise NCLErrorResult(result["error_message"])
print()
for day_entry in result["operational_stats"]:
print(day_entry["day"])
if day_entry["archive_success"] != 0:
print("{0:8} archive success".format(day_entry["archive_success"]))
print("{0:8} archive bytes".format(day_entry["success_bytes_in"]))
if day_entry["retrieve_success"] != 0:
print("{0:8} retrieve success".format(day_entry["retrieve_success"]))
print("{0:8} retrieve bytes".format(day_entry["success_bytes_out"]))
if day_entry["delete_success"] != 0:
print("{0:8} delete success".format(day_entry["delete_success"]))
if day_entry["listmatch_success"] != 0:
print("{0:8} listmatch success".format(day_entry["listmatch_success"]))
def _create_collection(args, identity, ncl_dict):
method = "POST"
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name, identity.auth_key,
identity.auth_key_id)
path = "/".join(["customers", identity.user_name, "collections"])
uri = compute_uri(path, action="create", name=ncl_dict["collection_name"])
response = http_connection.request(method,
uri,
body=None,
expected_status=CREATED)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
print(str(result))
def get_all_buckets(self):
method = "GET"
http_connection = HTTPConnection(
compute_default_hostname(),
self._config.user_name,
self._config.auth_key,
self._config.auth_key_id
)
uri = compute_uri(
"/".join(["customers", self._config.user_name, "collections"]),
)
self._log.info("requesting %s" % (uri, ))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError, instance:
self._log.error(str(instance))
http_connection.close()
raise
def _list_collections(args, identity, ncl_dict):
method = "GET"
if identity is None:
raise InvalidIdentity("Must have identity to list collections")
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name, identity.auth_key,
identity.auth_key_id)
path = "/".join(["customers", identity.user_name, "collections"])
uri = compute_uri(path)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
# TODO: add an option for verbose list
for entry in result:
print(entry["name"])
def _create_collection(args, identity, ncl_dict):
method = "POST"
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name,
identity.auth_key,
identity.auth_key_id)
path = "/".join(["customers", identity.user_name, "collections"])
uri = compute_uri(path, action="create", name=ncl_dict["collection_name"])
response = http_connection.request(method,
uri,
body=None,
expected_status=CREATED)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
print(str(result))
def _list_collections(args, identity, ncl_dict):
method = "GET"
if identity is None:
raise InvalidIdentity("Must have identity to list collections")
http_connection = HTTPConnection(compute_default_hostname(),
identity.user_name,
identity.auth_key,
identity.auth_key_id)
path = "/".join(["customers", identity.user_name, "collections"])
uri = compute_uri(path)
response = http_connection.request(method, uri, body=None)
data = response.read()
http_connection.close()
result = json.loads(data.decode("utf-8"))
# TODO: add an option for verbose list
for entry in result:
print(entry["name"])
def configure_access_control(self, access_control):
"""
set the bucket's access_control propoerty to a dict
"""
http_connection = HTTPConnection(
compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id
)
method = "PUT"
uri = compute_uri(
"/".join([
"customers",
self._identity.user_name,
"collections",
self._collection_name
]),
access_control="update"
)
body = None
headers = dict()
if access_control is not None:
body = access_control
headers["Content-Type"] = "application/json"
headers["Content-Length"] = len(body)
self._log.info("putting {0} {1}".format(uri, headers))
response = http_connection.request(method,
uri,
body=body,
headers=headers)
data = response.read()
http_connection.close()
return json.loads(data.decode("utf-8"))
def _list_keys(args, identity, ncl_dict):
method = "GET"
hostname = compute_collection_hostname(ncl_dict["collection_name"])
if identity is None:
http_connection = UnAuthHTTPConnection(hostname)
else:
http_connection = HTTPConnection(hostname, identity.user_name,
identity.auth_key,
identity.auth_key_id)
kwargs = {
"max_keys": _max_keys,
}
if "prefix" in ncl_dict and ncl_dict["prefix"] != "" and \
ncl_dict["prefix"] is not None:
kwargs["prefix"] = ncl_dict["prefix"]
if "marker" in ncl_dict and ncl_dict["marker"] != "" and \
ncl_dict["marker"] is not None:
kwargs["marker"] = ncl_dict["marker"]
if "delimiter" in ncl_dict and ncl_dict["delimiter"] != "" and \
ncl_dict["delimiter"] is not None:
kwargs["delimiter"] = ncl_dict["delimiter"]
uri = compute_uri("data/", **kwargs)
response = http_connection.request(method, uri)
data = response.read()
http_connection.close()
data_dict = json.loads(data.decode("utf-8"))
if "key_data" in data_dict:
for key_entry in data_dict["key_data"]:
print(key_entry["key"])
elif "prefixes" in data_dict:
for prefix in data_dict["prefixes"]:
print(prefix)
else:
raise ValueError("Unexpected return value {0}".format(data_dict))
def _retrieve_key(args, identity, ncl_dict):
method = "GET"
hostname = compute_collection_hostname(ncl_dict["collection_name"])
if identity is None:
http_connection = UnAuthHTTPConnection(hostname)
else:
http_connection = HTTPConnection(hostname, identity.user_name,
identity.auth_key,
identity.auth_key_id)
kwargs = {}
uri = compute_uri("data", ncl_dict["key"], **kwargs)
response = http_connection.request(method, uri, body=None)
while True:
data = response.read(_read_buffer_size)
if len(data) == 0:
break
sys.stdout.buffer.write(data)
http_connection.close()
def get_all_buckets(self):
"""
List all collections for the user
returns a list of motoboto.s3.Bucket objects
"""
method = "GET"
http_connection = HTTPConnection(
compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id
)
uri = compute_uri(
"/".join(["customers", self._identity.user_name, "collections"]),
)
self._log.info("requesting {0}".format(uri))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
self._log.info("reading response")
data = response.read()
http_connection.close()
collection_list = json.loads(data.decode("utf-8"))
bucket_list = list()
for collection_dict in collection_list:
bucket = Bucket(
self._identity,
collection_dict["name"],
versioning=collection_dict["versioning"]
)
bucket_list.append(bucket)
return bucket_list
def get_all_buckets(self):
"""
List all collections for the user
returns a list of motoboto.s3.Bucket objects
"""
method = "GET"
http_connection = HTTPConnection(compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id)
uri = compute_uri(
"/".join(["customers", self._identity.user_name, "collections"]), )
self._log.info("requesting {0}".format(uri))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
self._log.info("reading response")
data = response.read()
http_connection.close()
collection_list = json.loads(data.decode("utf-8"))
bucket_list = list()
for collection_dict in collection_list:
bucket = Bucket(self._identity,
collection_dict["name"],
versioning=collection_dict["versioning"])
bucket_list.append(bucket)
return bucket_list
def delete_bucket(self, bucket_name):
"""
remove (an empty) bucket from nimbus.io
This operation will fail if the collection contains any active keys.
When tis operaton succeeds, the colection/bucket name is available for
re-use.
"""
method = "DELETE"
http_connection = HTTPConnection(
compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id
)
if bucket_name.startswith("/"):
bucket_name = bucket_name[1:]
uri = compute_uri(
"/".join([
"customers",
self._identity.user_name,
"collections",
bucket_name
]),
)
self._log.info("requesting {0}".format(uri))
try:
response = http_connection.request(method, uri, body=None)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
response.read()
http_connection.close()
def create_bucket(self, bucket_name, access_control=None):
"""
create a nimbus.io collection, similar to an s3 bucket
nimbus.io organizes the objects that you store into collections. Every
nimbus.io key is a member of a collection. For efficient access to your
data nimbus.io uses the collection name as part of the `hostname`_.
For example, to act on objects in the collection
``my-temperature-readings``, your HTTP query would be directed to
hostname ``my-temperature-readings.nimbus.io``
This approach requires some restrictions on your collection names:
* collection names must be **unique**: you cannot use a colection name
that someone else is already using.
* Internet standards mandate that collection names may contain only
* the ASCII letters **a** through **z** (case-insensitive),
* the digits **0** through **9**,
* the hyphen (**-**).
* collection names must be between 1 and 63 characters long
nimbus.io gives you a default collection name of
``dd-<your user name>``
you don't need to create your default collection
you cannot delete your default collection
To reduce the inconvenience of creating a unique collection name,
nimbus.io provides a facility for creating guaranteed unique names of
the form ``rr-<your user-name>-<collection name>``. Of course, this
must comply with the restrictons mentioned above.
.. _hostname: http://en.wikipedia.org/wiki/Hostname
"""
method = "POST"
http_connection = HTTPConnection(compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id)
uri = compute_uri("/".join(
["customers", self._identity.user_name, "collections"]),
action="create",
name=bucket_name)
body = None
headers = dict()
if access_control is not None:
body = access_control
headers["Content-Type"] = "application/json"
headers["Content-Length"] = len(body)
self._log.info("requesting {0} {1}".format(uri, headers))
try:
response = http_connection.request(method,
uri,
body=body,
headers=headers,
expected_status=CREATED)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
response.read()
http_connection.close()
return Bucket(self._identity, bucket_name)
def create_bucket(self, bucket_name, access_control=None):
"""
create a nimbus.io collection, similar to an s3 bucket
nimbus.io organizes the objects that you store into collections. Every
nimbus.io key is a member of a collection. For efficient access to your
data nimbus.io uses the collection name as part of the `hostname`_.
For example, to act on objects in the collection
``my-temperature-readings``, your HTTP query would be directed to
hostname ``my-temperature-readings.nimbus.io``
This approach requires some restrictions on your collection names:
* collection names must be **unique**: you cannot use a colection name
that someone else is already using.
* Internet standards mandate that collection names may contain only
* the ASCII letters **a** through **z** (case-insensitive),
* the digits **0** through **9**,
* the hyphen (**-**).
* collection names must be between 1 and 63 characters long
nimbus.io gives you a default collection name of
``dd-<your user name>``
you don't need to create your default collection
you cannot delete your default collection
To reduce the inconvenience of creating a unique collection name,
nimbus.io provides a facility for creating guaranteed unique names of
the form ``rr-<your user-name>-<collection name>``. Of course, this
must comply with the restrictons mentioned above.
.. _hostname: http://en.wikipedia.org/wiki/Hostname
"""
method = "POST"
http_connection = HTTPConnection(
compute_default_hostname(),
self._identity.user_name,
self._identity.auth_key,
self._identity.auth_key_id
)
uri = compute_uri(
"/".join(["customers", self._identity.user_name, "collections"]),
action="create",
name=bucket_name
)
body = None
headers = dict()
if access_control is not None:
body = access_control
headers["Content-Type"] = "application/json"
headers["Content-Length"] = len(body)
self._log.info("requesting {0} {1}".format(uri, headers))
try:
response = http_connection.request(method,
uri,
body=body,
headers=headers,
expected_status=CREATED)
except LumberyardHTTPError:
instance = sys.exc_info()[1]
self._log.error(str(instance))
http_connection.close()
raise
response.read()
http_connection.close()
return Bucket(self._identity, bucket_name)
