def __init__(self, config):
'''*config* can be obtained from the function :func:`cloudfusion.store.sugarsync.sugarsync_store.SugarsyncStore.get_config`,
but you need to add user and password::
config = SugarsyncStore.get_config()
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
Or you can use a configuration file that already has password and username set by specifying a path::
path_to_my_config_file = '/home/joe/MySugarsync.ini'
config = get_config(path_to_my_config_file)
:param config: dictionary with key value pairs'''
#self.dir_listing_cache = {}
self._logging_handler = 'sugarsync'
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(self.logger)
manager = Manager()
self.path_cache = manager.dict()
# use a lock for synchronized appends
self._dir_listing_cache = manager.dict()
self._dir_listing_cache_lock = RLock()
self._last_partial_cache = manager.list()
self._time_of_last_partial_cache = 0
#error handling for authorization error
self.root = config["root"]
try:
self.client = SugarsyncClient(config)
except Exception, e:
raise StoreAutorizationError(repr(e), 0)
def __init__(self, config):
#self.dir_listing_cache = {}
self.robustness = 10
self._logging_handler = 'sugarsync'
self.logger = logging.getLogger(self._logging_handler)
self.path_cache = {}
self.root = config["root"]
self.client = SugarsyncClient(config)
self.time_difference = self._get_time_difference()
self.logger.debug("sugarsync store initialized")
super(SugarsyncStore, self).__init__()
class SugarsyncStore(Store):
def __init__(self, config):
#self.dir_listing_cache = {}
self.robustness = 10
self._logging_handler = 'sugarsync'
self.logger = logging.getLogger(self._logging_handler)
self.path_cache = {}
self.root = config["root"]
self.client = SugarsyncClient(config)
self.time_difference = self._get_time_difference()
self.logger.debug("sugarsync store initialized")
super(SugarsyncStore, self).__init__()
def get_name(self):
self.logger.debug("getting name")
return "Sugarsync"
def _translate_path(self, path):
self.logger.debug("translating path: "+path) #+" cache: "+str(self.path_cache)
path = to_unicode( path, "utf8")
if path in self.path_cache:
return self.path_cache[path]
if path == "/":
return self.root
else:
parent_dir = to_unicode( os.path.dirname(path), "utf8")
self.path_cache[parent_dir] = self._translate_path(parent_dir)
collection = self._parse_collection(self.path_cache[parent_dir])
for item in collection:
if parent_dir[-1] != "/":
parent_dir += "/"
self.path_cache[ parent_dir+to_unicode( item["name"], "utf8") ] = item["reference"]
if not path in self.path_cache:
self.logger.warn("could not translate path: " +path)
raise NoSuchFilesytemObjectError(path,0)
return self.path_cache[path]
def _parse_collection(self, translated_path):
""":returns: dict a dictionary with all paths of the collection at :param:`translated_path` as keys and the corresponding nested dictionaries with the key/value pair for is_dir and reference."""
ret = []
resp = self.client.get_dir_listing(translated_path)
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not get directory listing: " +translated_path+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
raise NoSuchFilesytemObjectError(translated_path, resp.status)
xml_tree = dom.parseString(resp.data)
for collection in xml_tree.documentElement.getElementsByTagName("collection"):
item = {}
item["is_dir"] = collection.getAttribute("type") == "folder"
item["name"] = collection.getElementsByTagName("displayName")[0].firstChild.nodeValue.encode('utf8')
reference_url = collection.getElementsByTagName("ref")[0].firstChild.nodeValue
reference_uid = regSearchString('.*:(.*)', reference_url)
item["reference"] = reference_uid
ret.append(item)
for collection in xml_tree.documentElement.getElementsByTagName("file"):
item = {}
item["is_dir"] = False
item["name"] = collection.getElementsByTagName("displayName")[0].firstChild.nodeValue
item["size"] = collection.getElementsByTagName("size")[0].firstChild.nodeValue
item["lastModified"] = collection.getElementsByTagName("lastModified")[0].firstChild.nodeValue
item["presentOnServer"] = collection.getElementsByTagName("lastModified")[0].firstChild.nodeValue
reference_url = collection.getElementsByTagName("ref")[0].firstChild.nodeValue
reference_uid = regSearchString('.*:(.*)', reference_url)
item["reference"] = reference_uid
ret.append(item)
return ret
def account_info(self):
self.logger.debug("retrieving account info")
info = self.client.user_info()
partial_tree = {"user": {"quota": {"limit": "limit", "usage": "usage"}}}
DictXMLParser().populate_dict_with_XML_leaf_textnodes(info.data, partial_tree)
#print response.status, response.reason, response.getheaders()
ret = {}
ret['overall_space'] = int(partial_tree['user']['quota']['limit'])
ret['used_space'] = int(partial_tree['user']['quota']['usage'])
return "Sugarsync overall space: %s, used space: %s" % (ret['overall_space'], ret['used_space'])
def get_overall_space(self):
self.logger.debug("retrieving all space")
info = self.client.user_info()
partial_tree = {"user": {"quota": {"limit": "limit", "usage": "usage"}}}
DictXMLParser().populate_dict_with_XML_leaf_textnodes(info.data, partial_tree)
#print response.status, response.reason, response.getheaders()
if info.status <200 or info.status >= 300:
self.logger.warn("could not retrieve overall space"+"\nstatus: %s reason: %s" % (info.status, info.reason))
return int(partial_tree['user']['quota']['limit'])
def get_used_space(self):
self.logger.debug("retrieving used space")
info = self.client.user_info()
partial_tree = {"user": {"quota": {"limit": "limit", "usage": "usage"}}}
DictXMLParser().populate_dict_with_XML_leaf_textnodes(info.data, partial_tree)
#print response.status, response.reason, response.getheaders()
if info.status <200 or info.status >= 300:
self.logger.warn("could not retrieve used space"+"\nstatus: %s reason: %s" % (info.status, info.reason))
return int(partial_tree['user']['quota']['usage'])
def get_file(self, path_to_file):
self.logger.debug("getting file: " +path_to_file)
self._raise_error_if_invalid_path(path_to_file)
file = self.client.get_file( self._translate_path(path_to_file) )
if file.status <200 or file.status >= 300:
self.logger.warn("could not get file: %s\nstatus: %s reason: %s" % (path_to_file, file.status, file.reason))
if file.status == 401 or file.status >= 500:
self._reconnect()
self.robustness -= 1
if self.robustness > 0:
self.logger.info("retrying")
return self.get_file(path_to_file)
return file.data
def store_fileobject(self, fileobject, path_to_file):
self.logger.debug("storing file object to "+path_to_file)
if not self.exists(path_to_file):
self._create_file(path_to_file)
resp = self.client.put_file( fileobject, self._translate_path(path_to_file) )
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not store file to " +path_to_file+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
self.robustness -= 1
if self.robustness > 0:
self.logger.info("retrying")
self.store_fileobject(fileobject, path_to_file)
def _create_file(self, path, mime='text/x-cloudfusion'):
self.logger.debug("creating file object "+path)
name = os.path.basename(path)
directory = os.path.dirname(path)
translated_dir = self._translate_path(directory)
resp = self.client.create_file(translated_dir, name)
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not create file " +path+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
def delete(self, path):
self.logger.debug("deleting " +path)
if path == "/":
return
if path[-1] == "/":
path = path[0:-1]
self._raise_error_if_invalid_path(path)
resp = self.client.delete_file( self._translate_path(path) )
if resp.status <200 or resp.status >= 300:
resp = self.client.delete_folder( self._translate_path(path) )
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not delete " +path+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
else:
del self.path_cache[path]
return resp.status
def create_directory(self, path):
self.logger.debug("creating directory " +path)
self._raise_error_if_invalid_path(path)
if path == "/":
return
if path[-1] == "/":
path = path[0:-1]
if self.exists(path):
raise AlreadyExistsError("directory %s does already exist"%path, 401)
name = os.path.basename(path)
directory = os.path.dirname(path)
resp = self.client.create_folder( self._translate_path(directory), name )
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not create directory: " +path+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
return resp.status
def get_directory_listing(self, directory):
self.logger.debug("getting directory listing for "+directory)
ret = []
translated_dir = self._translate_path(directory)
collection = self._parse_collection(translated_dir)
if directory[-1] != "/":
directory += "/"
for item in collection:
ret.append( directory+item['name'] )
#self.logger.warn(str(ret))
return ret
def duplicate(self, path_to_src, path_to_dest): #src might be a directory
self.logger.debug("duplicating " +path_to_src+" to "+path_to_dest)
self._raise_error_if_invalid_path(path_to_src)
self._raise_error_if_invalid_path(path_to_dest)
if path_to_src[-1] == "/":
path_to_src = path_to_src[0:-1]
if path_to_dest[-1] == "/":
path_to_dest = path_to_dest[0:-1]
dest_name = os.path.basename(path_to_dest)
dest_dir = os.path.dirname(path_to_dest)
if path_to_dest.startswith(path_to_src) and dest_name == os.path.basename(path_to_src):
self.logger.warning("cannot copy folder to itself")
return #DBG raise error
translated_dest_dir = self._translate_path( dest_dir )
translated_src = self._translate_path(path_to_src)
if self.is_dir(path_to_src):
#make destination directory: (might exist)
if not self.exists(path_to_dest):
self.create_directory(path_to_dest)
#copy all files from original directory:
for item in self._parse_collection(translated_src):
if item['is_dir']:#copy all folders form original directory
self.duplicate(path_to_src+"/"+item['name'], path_to_dest+"/"+item['name'])
else:
resp = self.client.duplicate_file(item['reference'], translated_dest_dir, dest_name)
if resp.status != 200:
self.logger.warn("could not duplicate " +path_to_src+" to "+path_to_dest+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
else:
#if dest exists remove
if self.exists(path_to_dest):
self.delete(path_to_dest)
resp = self.client.duplicate_file(translated_src, translated_dest_dir, dest_name)
if resp.status < 200 or resp.status >= 300:
self.logger.warn("could not duplicate " +path_to_src+" to "+path_to_dest+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
if resp.status == 401 or resp.status >= 500:
self._reconnect()
def _get_metadata(self, path):
self.logger.debug("getting metadata for "+path)
self._raise_error_if_invalid_path(path)
if path == "/": # workaraund for root metadata necessary for sugarsync?
ret = {}
ret["bytes"] = 0
ret["modified"] = time.time()
ret["path"] = "/"
ret["is_dir"] = True
return ret;
if path[-1] == "/":
path = path[0:-1]
is_file = True
p=None; resp=None;
try:
p = self._translate_path(path)
except Exception as e:
self.logger.debug("getting metadata for "+path+" failed with: "+str(e))
try:
resp = self.client.get_file_metadata( p )
except Exception as e:
self.logger.debug("getting metadata 5 for "+path+" failed with: "+str(e))
if resp.status <200 or resp.status >= 300:
is_file = False
resp = self.client.get_folder_metadata( self._translate_path(path) )
if resp.status == 401 or resp.status >= 500:
self._reconnect()
if resp.status <200 or resp.status >= 300:
self.logger.warn("could not get metadata: " +path+"\nstatus: %s reason: %s" % (resp.status, resp.reason))
raise NoSuchFilesytemObjectError(path, resp.status)
ret = {}
if is_file:
partial_tree = {"file": {"size": "", "lastModified": "", "timeCreated": ""}}
DictXMLParser().populate_dict_with_XML_leaf_textnodes(resp.data, partial_tree)
ret["bytes"] = int(partial_tree['file']['size'])
try:#"Sat, 21 Aug 2010 22:31:20 +0000"#2011-05-10T06:18:33.000-07:00 Time conversion error: 2011-05-20T05:15:44.000-07:00
lastModified = partial_tree['file']['lastModified']
modified = time.mktime( time.strptime( lastModified[0:-6], "%Y-%m-%dT%H:%M:%S.000") )
time_offset = time.strptime( lastModified[-5:], "%H:%M")
time_delta = 60*60*time_offset.tm_hour + 60*time_offset.tm_min
modified += time_delta
ret["modified"] = modified - self.time_difference
except Exception as x:
self.logger.warn("Time conversion error: %s reason: %s" % ( str(partial_tree['file']['lastModified']), str(x)) )
raise DateParseError("Error parsing modified attribute: %s" % str(x));
ret["created"] = partial_tree['file']['timeCreated']
ret["path"] = path
ret["is_dir"] = False
else:
partial_tree = {"folder": {"timeCreated": ""}}
DictXMLParser().populate_dict_with_XML_leaf_textnodes(resp.data, partial_tree)
ret["bytes"] = 0
ret["modified"] = time.time()
ret["created"] = partial_tree['folder']['timeCreated']
ret["path"] = path
ret["is_dir"] = True
return ret;
def _get_time_difference(self):
resp = self.client.user_info()
return time.mktime( time.strptime(resp.getheader('date'), "%a, %d %b %Y %H:%M:%S GMT") ) - time.time()
def _reconnect(self):
self.client._reconnect()
def get_logging_handler(self):
return self._logging_handler
