def crawler(q, errorlogger):
api = EsuRestApi(host, port, uid, secret)
def logexception(problematic_item, problem):
print problematic_item, problem
log = problematic_item + str(problem)
errorlogger.put(log)
while 1:
if not q.empty():
task = str(q.get())
if task == "bye":
break
task = task
path = write_location + str(task)
print q.qsize(), current_process(), "Downloading ", path
try:
data = api.read_object(task)
with open(path, "w") as file:
file.write(data)
except EsuException:
logexception(task, ["EsuExc"])
continue
except:
logexception(task, ["UnknownExc"])
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
#URL string: atmos://host/path/
self.url_string = duplicity.backend.strip_auth_from_url(
self.parsed_url)
self.url_path = '/' + '/'.join(self.url_string.split('/')[3:])
host = self.url_string.split('/')[2].split(':')[0]
#Hacks
try:
port = self.url_string.split('/')[2].split(':')[1]
except Exception:
port = 443
pass
parser = SafeConfigParser()
parser.read('/etc/duplicity/atmos.ini')
uid = parser.get(host, 'uid')
secret = parser.get(host, 'secret')
log.Debug("Parsed URL:" + self.url_string)
#Init Atmos connection
self.api = EsuRestApi(host, int(port), uid, secret)
# Use an explicit directory name.
if self.url_string[-1] != '/':
self.url_string += '/'
def crawler(q, errorlogger):
api = EsuRestApi(host, port, uid, secret)
while True:
def logexception(problematic_item, problem):
print problematic_item, problem
log = problematic_item + [problem]
errorlogger.put(log)
if not q.empty():
task = q.get()
typeof, item_uid, path = task[0], task[1], task[2]
if typeof == "regular":
#download the item
uri = write_location + path
print q.qsize(), current_process(), "Downloading ", path
with open(uri, "w") as file:
try:
file.write(api.read_object(item_uid))
except EsuException:
logexception(task, ["EsuExp"])
continue
elif typeof == "directory":
#explore the directory
path = path + "/"
print q.qsize(), current_process(), "exploring :", path
folder = write_location + path.rsplit('/', 1)[0] + "/"
print folder
if not exists(folder):
makedirs(folder)
token = None
# if more than 10000 files in the dir, a token is returned for pagination
while True:
try:
reply = api.list_directory(path, token=token)
list_of_items, token = reply[0], reply[1]
except EsuException:
logexception(task, ["EsuExp"])
else:
for item in list_of_items:
res_uid, res_name, res_typeof = item[0], item[
2], item[1]
fullpath = path + res_name
q.put([res_typeof, res_uid, fullpath])
if token == None:
break
elif typeof == "bye":
break
else:
# this should not happen
msg = typeof + " is not a known type. "
raise Exception(msg)
continue
def setUp(self):
self.esu = EsuRestApi(self.host, self.port, self.uid, self.secret)
self.oid_clean_up = []
self.path_clean_up = []
ATMOS_KEY = "194ed3dd98414dcebc780f030d36e616/[email protected]" ATMOS_SECRET = "/rDrx8U/Os8KJsbLClrLrDrDuyyV74YbBFrHfm0d" HOST = 'atmos.ecstestdrive.com' PORT = 443 import re import os import sys import subprocess import time import json from datetime import datetime from EsuRestApi import EsuRestApi import threading api = EsuRestApi(HOST, PORT, ATMOS_KEY, ATMOS_SECRET) results = {} objectList = [] fileDetails = {} fileDetails['../512k'] = {"count": '100', "size": '512'} fileDetails['../1MB'] = {"count": '50', "size": '1000'} fileDetails['../10MB'] = {"count": '25', "size": '10000'} fileDetails['../100MB'] = {"count": '10', "size": '100000'} fileDetails['../1000MB'] = {"count": '5', "size": '1000000'} transferTimeList = [] throughputList = [] numberOfIterations = 1 threads = [] maxthreads = 1 sema = threading.Semaphore(value=maxthreads) threads = list()
