def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
#URL string: atmos://host/path/
self.url_string = duplicity.backend.strip_auth_from_url(
self.parsed_url)
self.url_path = '/' + '/'.join(self.url_string.split('/')[3:])
host = self.url_string.split('/')[2].split(':')[0]
#Hacks
try:
port = self.url_string.split('/')[2].split(':')[1]
except Exception:
port = 443
pass
parser = SafeConfigParser()
parser.read('/etc/duplicity/atmos.ini')
uid = parser.get(host, 'uid')
secret = parser.get(host, 'secret')
log.Debug("Parsed URL:" + self.url_string)
#Init Atmos connection
self.api = EsuRestApi(host, int(port), uid, secret)
# Use an explicit directory name.
if self.url_string[-1] != '/':
self.url_string += '/'
def crawler(q, errorlogger):
api = EsuRestApi(host, port, uid, secret)
def logexception(problematic_item, problem):
print problematic_item, problem
log = problematic_item + str(problem)
errorlogger.put(log)
while 1:
if not q.empty():
task = str(q.get())
if task == "bye":
break
task = task
path = write_location + str(task)
print q.qsize(), current_process(), "Downloading ", path
try:
data = api.read_object(task)
with open(path, "w") as file:
file.write(data)
except EsuException:
logexception(task, ["EsuExc"])
continue
except:
logexception(task, ["UnknownExc"])
def crawler(q, errorlogger):
api = EsuRestApi(host, port, uid, secret)
while True:
def logexception(problematic_item, problem):
print problematic_item, problem
log = problematic_item + [problem]
errorlogger.put(log)
if not q.empty():
task = q.get()
typeof, item_uid, path = task[0], task[1], task[2]
if typeof == "regular":
#download the item
uri = write_location + path
print q.qsize(), current_process(), "Downloading ", path
with open(uri, "w") as file:
try:
file.write(api.read_object(item_uid))
except EsuException:
logexception(task, ["EsuExp"])
continue
elif typeof == "directory":
#explore the directory
path = path + "/"
print q.qsize(), current_process(), "exploring :", path
folder = write_location + path.rsplit('/', 1)[0] + "/"
print folder
if not exists(folder):
makedirs(folder)
token = None
# if more than 10000 files in the dir, a token is returned for pagination
while True:
try:
reply = api.list_directory(path, token=token)
list_of_items, token = reply[0], reply[1]
except EsuException:
logexception(task, ["EsuExp"])
else:
for item in list_of_items:
res_uid, res_name, res_typeof = item[0], item[
2], item[1]
fullpath = path + res_name
q.put([res_typeof, res_uid, fullpath])
if token == None:
break
elif typeof == "bye":
break
else:
# this should not happen
msg = typeof + " is not a known type. "
raise Exception(msg)
continue
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
#URL string: atmos://host/path/
self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
self.url_path = '/' + '/'.join(self.url_string.split('/')[3:])
host = self.url_string.split('/')[2].split(':')[0]
#Hacks
try:
port = self.url_string.split('/')[2].split(':')[1]
except Exception:
port=443
pass
parser = SafeConfigParser()
parser.read('/etc/duplicity/atmos.ini')
uid=parser.get(host, 'uid')
secret=parser.get(host, 'secret')
log.Debug("Parsed URL:" + self.url_string)
#Init Atmos connection
self.api = EsuRestApi( host, int(port), uid, secret )
# Use an explicit directory name.
if self.url_string[-1] != '/':
self.url_string += '/'
class AtmosBackend(duplicity.backend.Backend):
"""Connect to remote store using File Transfer Protocol"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
#URL string: atmos://host/path/
self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
self.url_path = '/' + '/'.join(self.url_string.split('/')[3:])
host = self.url_string.split('/')[2].split(':')[0]
#Hacks
try:
port = self.url_string.split('/')[2].split(':')[1]
except Exception:
port=443
pass
parser = SafeConfigParser()
parser.read('/etc/duplicity/atmos.ini')
uid=parser.get(host, 'uid')
secret=parser.get(host, 'secret')
log.Debug("Parsed URL:" + self.url_string)
#Init Atmos connection
self.api = EsuRestApi( host, int(port), uid, secret )
# Use an explicit directory name.
if self.url_string[-1] != '/':
self.url_string += '/'
def put(self, source_path, remote_filename = None):
"""Transfer source_path to remote_filename"""
if not remote_filename:
remote_filename = source_path.get_filename()
remote_path = self.url_path + '/' + remote_filename
log.Debug("Upload " + source_path.get_filename() + " to " + remote_path)
object_id = self.api.create_object_on_path(remote_path, None, None, None, None, open(source_path.name).read())
def get(self, remote_filename, local_path):
"""Get remote filename, saving it to local_path"""
remote_path = os.path.join(urllib.unquote(self.parsed_url.path), remote_filename).rstrip()
log.Debug("Download " + remote_filename + " to " + local_path.name)
f = open(local_path.name, 'w')
f.write(self.api.read_object_from_path(self.url_path + '/' + remote_filename))
f.close()
def list(self):
"""List files in directory"""
# Do a long listing to avoid connection reset
log.Debug("Listing " + self.url_path)
listing = []
try:
for file in self.api.list_directory(self.url_path)[0]:
log.Debug("Found: " + file[2])
listing.append (file[2])
except Exception:
#The library will fail with EsuException: EsuException when remote folder doesn't exist
pass
return listing
def delete(self, filename_list):
"""Delete files in filename_list"""
for filename in filename_list:
filename=self.url_path + '/' + filename
for fileinfo in self.api.list_directory(os.path.dirname(filename))[0]:
if fileinfo[2] == os.path.basename(filename):
file_id = fileinfo[0]
break
log.Debug("Delete " + filename + "(ID: " + file_id + ")")
self.api.delete_object(file_id)
def setUp(self):
self.esu = EsuRestApi(self.host, self.port, self.uid, self.secret)
self.oid_clean_up = []
self.path_clean_up = []
class EsuRestApiTest(unittest.TestCase):
# Enter your own host in the form of sub.domain.com or 10.0.1.250
host = "lciga090.lss.emc.com"
# Enter the port where Atmos lives here
port = 80
# Enter your full UID in the form of something/something_else
uid = "0e2200283d4143d9b2895992a64cd319/test"
# Enter your secret here. (shhsh!)
secret = "lYp88RptTEnBOEh/DC0w5ys7olU="
def setUp(self):
self.esu = EsuRestApi(self.host, self.port, self.uid, self.secret)
self.oid_clean_up = []
self.path_clean_up = []
def tearDown(self):
if self.oid_clean_up:
for object in self.oid_clean_up:
self.esu.delete_object(object)
if self.path_clean_up:
dir = self.path_clean_up[0].split("/")
self.esu.delete_directory(dir[0])
def test_create_empty_object(self):
data = " "
oid = self.esu.create_object(data=data)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
self.oid_clean_up.append(oid)
def test_create_empty_object_on_path(self):
data = " "
path = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8)) + "/file.data"
oid = self.esu.create_object_on_path(data=data, path=path)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
self.oid_clean_up.append(oid)
self.path_clean_up.append(path)
def test_create_object_with_content(self):
data = "The quick brown fox jumps over the lazy dog"
oid = self.esu.create_object(data=data)
object = self.esu.read_object(oid)
self.assertEquals(data, object)
self.oid_clean_up.append(oid)
def test_create_object_with_content_and_checksum(self):
data = "The quick brown fox jumps over the lazy dog"
checksum = "SHA1/%d/%s" % (len(data), hashlib.sha1(data).hexdigest())
oid = self.esu.create_object(data=data, checksum=checksum)
self.oid_clean_up.append(oid)
object = self.esu.read_object(oid)
self.assertEquals(data, object)
def test_create_object_on_path_with_content(self):
data = "The quick brown fox jumps over the lazy dog"
path = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8)) + "/file.data"
oid = self.esu.create_object_on_path(data=data, path=path)
self.oid_clean_up.append(oid)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
def test_create_object_on_path_with_content_and_checksum(self):
data = "The quick brown fox jumps over the lazy dog"
path = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8)) + "/file.data"
checksum = "SHA1/%d/%s" % (len(data), hashlib.sha1(data).hexdigest())
oid = self.esu.create_object_on_path(data=data, path=path, checksum=checksum)
self.oid_clean_up.append(oid)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
def test_create_object_on_path_with_metadata(self):
data = "The quick brown fox jumps over the lazy dog"
listable_meta = {"key1" : "value1", "key2" : "value2", "key3" : "value3"}
path = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8)) + "/file.data"
oid = self.esu.create_object_on_path(data=data, path=path, listable_meta=listable_meta)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
# Retrieves existing metadata for an object and compares it to the known metadata dictionary that was stored
metadata = self.esu.get_user_metadata(oid)['listable_user_meta']
self.assertEqual(listable_meta, metadata, "metadata key/values are wrong")
self.oid_clean_up.append(oid)
self.path_clean_up.append(path)
def test_create_object_with_metadata(self):
data = "The quick brown fox jumps over the lazy dog"
listable_meta = {"key1" : "value1", "key2" : "value2", "key3" : "value3"}
oid = self.esu.create_object(data=data, listable_meta=listable_meta)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
# Retrieves existing metadata for an object and compares it to the known metadata dictionary that was stored
metadata = self.esu.get_user_metadata(oid)['listable_user_meta']
self.assertEqual(listable_meta, metadata, "metadata key/values are wrong")
self.oid_clean_up.append(oid)
def test_read_acl(self):
data = "The quick brown fox jumps over the lazy dog"
oid = self.esu.create_object(data=data)
uid = self.esu.uid.split("/")[0]
user_acl = "%s=FULL_CONTROL" % uid
resp = self.esu.set_acl(oid, user_acl)
acl = self.esu.get_acl(oid)['user_acl'][uid]
self.assertEqual(acl, "FULL_CONTROL", "acl does not match")
self.oid_clean_up.append(oid)
def test_delete_user_metadata(self):
data = "The quick brown fox jumps over the lazy dog"
listable_meta = {"key1" : "value1"}
oid = self.esu.create_object(data=data, listable_meta=listable_meta)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
# Retrieves existing metadata for an object and compares it to the known metadata dictionary that was stored
metadata = self.esu.get_user_metadata(oid)['listable_user_meta']
self.assertEqual(listable_meta, metadata, "metadata key/values are wrong")
self.esu.delete_user_metadata(object_id=oid, metadata_key="key1")
metadata = self.esu.get_user_metadata(oid)['listable_user_meta']
self.assertEqual(metadata, {})
self.oid_clean_up.append(oid)
def test_get_system_metadata(self):
data = "The quick brown fox jumps over the lazy dog"
oid = self.esu.create_object(data=data)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
system_meta = self.esu.get_system_metadata(oid)
self.assertTrue(system_meta['size'], "Size should be > 0" )
self.assertTrue(system_meta['ctime'], "the ctime was not set")
self.assertEqual(system_meta['objectid'], oid, "Object IDs do not match")
self.oid_clean_up.append(oid)
def test_list_objects(self):
data = "The quick brown fox jumps over the lazy dog"
key = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8))
listable_meta = {key : "value1"}
oid = self.esu.create_object(data=data, listable_meta=listable_meta)
self.assertTrue(oid, "null object ID returned")
object = self.esu.read_object(oid)
self.assertEqual(object, data, "wrong object content")
list = self.esu.list_objects(metadata_key=key)
self.assertEqual(oid, list[0][0], "wrong object ids")
self.oid_clean_up.append(oid)
def test_list_directory(self):
data = "The quick brown fox jumps over the lazy dog"
path = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(8)) + "/file.data"
oid = self.esu.create_object_on_path(data=data, path=path)
dir = path.split("/")[0]
list = self.esu.list_directory(dir)
self.assertEqual(oid, list[0][0][0], "wrong object ids")
self.oid_clean_up.append(oid)
self.path_clean_up.append(path)
def test_delete_object(self):
data = "The quick brown fox jumps over the lazy dog"
oid = self.esu.create_object(data=data)
self.assertTrue(oid, "null object ID returned")
self.esu.delete_object(oid)
try:
object = self.esu.read_object(oid)
except EsuException, e:
self.assertEqual(e.atmos_error_code, "1003", "wrong error code")
ATMOS_KEY = "194ed3dd98414dcebc780f030d36e616/[email protected]" ATMOS_SECRET = "/rDrx8U/Os8KJsbLClrLrDrDuyyV74YbBFrHfm0d" HOST = 'atmos.ecstestdrive.com' PORT = 443 import re import os import sys import subprocess import time import json from datetime import datetime from EsuRestApi import EsuRestApi import threading api = EsuRestApi(HOST, PORT, ATMOS_KEY, ATMOS_SECRET) results = {} objectList = [] fileDetails = {} fileDetails['../512k'] = {"count": '100', "size": '512'} fileDetails['../1MB'] = {"count": '50', "size": '1000'} fileDetails['../10MB'] = {"count": '25', "size": '10000'} fileDetails['../100MB'] = {"count": '10', "size": '100000'} fileDetails['../1000MB'] = {"count": '5', "size": '1000000'} transferTimeList = [] throughputList = [] numberOfIterations = 1 threads = [] maxthreads = 1 sema = threading.Semaphore(value=maxthreads) threads = list()
class AtmosBackend(duplicity.backend.Backend):
"""Connect to remote store using File Transfer Protocol"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
#URL string: atmos://host/path/
self.url_string = duplicity.backend.strip_auth_from_url(
self.parsed_url)
self.url_path = '/' + '/'.join(self.url_string.split('/')[3:])
host = self.url_string.split('/')[2].split(':')[0]
#Hacks
try:
port = self.url_string.split('/')[2].split(':')[1]
except Exception:
port = 443
pass
parser = SafeConfigParser()
parser.read('/etc/duplicity/atmos.ini')
uid = parser.get(host, 'uid')
secret = parser.get(host, 'secret')
log.Debug("Parsed URL:" + self.url_string)
#Init Atmos connection
self.api = EsuRestApi(host, int(port), uid, secret)
# Use an explicit directory name.
if self.url_string[-1] != '/':
self.url_string += '/'
def put(self, source_path, remote_filename=None):
"""Transfer source_path to remote_filename"""
if not remote_filename:
remote_filename = source_path.get_filename()
remote_path = self.url_path + '/' + remote_filename
log.Debug("Upload " + source_path.get_filename() + " to " +
remote_path)
object_id = self.api.create_object_on_path(
remote_path, None, None, None, None,
open(source_path.name).read())
def get(self, remote_filename, local_path):
"""Get remote filename, saving it to local_path"""
remote_path = os.path.join(urllib.unquote(self.parsed_url.path),
remote_filename).rstrip()
log.Debug("Download " + remote_filename + " to " + local_path.name)
f = open(local_path.name, 'w')
f.write(
self.api.read_object_from_path(self.url_path + '/' +
remote_filename))
f.close()
def list(self):
"""List files in directory"""
# Do a long listing to avoid connection reset
log.Debug("Listing " + self.url_path)
listing = []
try:
for file in self.api.list_directory(self.url_path)[0]:
log.Debug("Found: " + file[2])
listing.append(file[2])
except Exception:
#The library will fail with EsuException: EsuException when remote folder doesn't exist
pass
return listing
def delete(self, filename_list):
"""Delete files in filename_list"""
for filename in filename_list:
filename = self.url_path + '/' + filename
for fileinfo in self.api.list_directory(
os.path.dirname(filename))[0]:
if fileinfo[2] == os.path.basename(filename):
file_id = fileinfo[0]
break
log.Debug("Delete " + filename + "(ID: " + file_id + ")")
self.api.delete_object(file_id)