def _request(self, url, params, retries=3, is_cert=False):
if not isinstance(params, dict):
raise Exception('params must be a dict')
params['sign'] = self.build_sign(**params)
xml_str = self.to_xml(**params)
ret = {}
error = None
for i in xrange(retries):
try:
if is_cert:
r = requests.post(url,
data=smart_str(xml_str),
cert=self.cert)
else:
r = requests.post(url, data=smart_str(xml_str))
for child in ET.fromstring(smart_str(r.text)):
ret[child.tag] = child.text
return ret
except Exception as e:
error = e
continue
raise 'Failed to request url, %s' % error
def __init__(self, key, secret, host, app={}):
""" if app = int return app(id)
elif app = {} init app with dict
new app using app = {}, id = 0
"""
self.host = host
self.consumer_key = key
self.consumer_secret = secret
if type(app) == int:
self.id = app
self.refresh()
else:
## can't modify
self.id = int(app.get("id", 0))
self.created = app.get("created", "")
self.deleted = app.get("deleted", "")
self.last_saved = app.get("last_saved", "")
self.socialize_consumer_key = app.get("socialize_consumer_key", "")
self.socialize_consumer_secret = app.get("socialize_consumer_secret", "")
self.socialize_app = app.get("socialize_app", "")
self.push_certificate = app.get("push_certificate", None)
self.is_socialize_editable = app.get("is_socialize_editable", True)
self.c2dm_token_source = app.get("c2dm_token_source", "socialize")
self.notification_quotas = app.get("quotas", {})
self.__update_notification_quotas()
## modifiable
notifications_enabled = app.get("notifications_enabled", False)
if type(notifications_enabled) == str:
if notifications_enabled == "false":
notifications_enabled = False
else:
notifications_enabled = True
self.notifications_enabled = notifications_enabled
self.android_package_name = app.get("android_package_name", "")
self.c2dm_sender_auth_token = app.get("c2dm_sender_auth_token", "")
self.apple_store_id = app.get("apple_store_id", "")
self.category = app.get("category", "")
self.description = smart_str(app.get("description", ""), strings_only=True)
self.name = smart_str(app.get("name", ""), strings_only=True)
self.mobile_platform = app.get("platforms", [])
self.resource_uri = app.get("resource_uri", "")
self.stats = app.get("stats", "")
self.display_name = self.name
self.icon_url = app.get("icon_url", None)
self.stats = app.get("stats", {})
if len(self.stats) > 0:
self.__calculate_stats(self.stats)
# logger.info(self.stats)
# to make forward and backward compatible with API-user changes (temporary)
user_id = int(app.get("user", "0"))
if user_id == 0:
user_id = int(app.get("user_id", "0"))
self.user = user_id
def _request(self, url, params, retries=3, is_cert=False):
if not isinstance(params, dict):
raise Exception('params must be a dict')
params['sign'] = self.build_sign(**params)
xml_str = self.to_xml(**params)
ret = {}
error = None
for i in xrange(retries):
try:
if is_cert:
r = requests.post(url, data=smart_str(xml_str),
cert=self.cert)
else:
r = requests.post(url, data=smart_str(xml_str))
for child in ET.fromstring(smart_str(r.text)):
ret[child.tag] = child.text
return ret
except Exception as e:
error = e
continue
raise 'Failed to request url, %s' % error
def listdir_container(self, cache, container, path=""):
"""Fills cache with the list dir of the container"""
container = smart_str(container)
path = smart_str(path)
logging.debug("listdir container %r path %r" % (container, path))
if path:
prefix = path.rstrip("/")+"/"
else:
prefix = None
_, objects = self.conn.get_container(container, prefix=prefix, delimiter="/")
# override 10000 objects limit with markers
nbobjects = len(objects)
while nbobjects >= 10000:
# get last object as a marker
lastobject = objects[-1]
if 'subdir' in lastobject:
# {u'subdir': 'dirname'}
lastobjectname = lastobject['subdir'].rstrip("/")
else:
lastobjectname = lastobject['name']
# get a new list with the marker
_, newobjects = self.conn.get_container(container, prefix=prefix, delimiter="/", marker=lastobjectname)
# get the new list length
nbobjects = len(newobjects)
logging.debug("number of objects after marker %s: %s" % (lastobjectname, nbobjects))
# add the new list to current list
objects.extend(newobjects)
logging.debug("total number of objects %s:" % len(objects))
for obj in objects:
# {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'},
if 'subdir' in obj:
# {u'subdir': 'dirname'}
obj['name'] = obj['subdir'].rstrip("/")
elif obj.get('bytes') == 0 and obj.get('hash') and obj.get('content_type') != 'application/directory':
# if it's a 0 byte file, has a hash and is not a directory, we make an extra call
# to check if it's a manifest file and retrieve the real size / hash
manifest_obj = self.conn.head_object(container, obj['name'])
logging.debug("possible manifest file: %r" % manifest_obj)
if 'x-object-manifest' in manifest_obj:
logging.debug("manifest found: %s" % manifest_obj['x-object-manifest'])
obj['hash'] = manifest_obj['etag']
obj['bytes'] = int(manifest_obj['content-length'])
obj['count'] = 1
# Keep all names in utf-8, just like the filesystem
name = posixpath.basename(obj['name']).encode("utf-8")
cache[name] = self._make_stat(**obj)
def get_projects(self, bot, event):
if not self.projects:
json = self.get(self.base_url + "projects.json")
self.projects = json
else:
json = self.projects
bot.connection.privmsg(bot.main_channel, "Sir, you have %d projects in basecamp right now:" % len(json))
for count, project in enumerate(json):
sleep(1) # needed to not flood the server
name = "%s: %s" % (smart_str(project["name"]), smart_str(project["description"]))
bot.connection.privmsg(bot.main_channel, "%d: %s" % (count + 1, name))
bot.connection.privmsg(
bot.main_channel, "That was all your projects in basecamp. Do you need to know more ? Just ask."
)
def key(self, index):
"""Returns a key for a user distributed cache."""
tenant_name = self.cffs.tenant_name or "-"
logging.debug("cache key for %r" % [self.cffs.authurl, self.cffs.username, tenant_name, index])
if not hasattr(self, "_key_base"):
self._key_base = md5("%s%s%s" % (self.cffs.authurl, self.cffs.username, tenant_name)).hexdigest()
return "%s-%s" % (self._key_base, md5(smart_str(index)).hexdigest())
def key(self, index):
"""Returns a key for a user distributed cache."""
tenant_name = self.cffs.tenant_name or "-"
logging.debug("cache key for %r" % [self.cffs.authurl, self.cffs.username, tenant_name, index])
if not hasattr(self, "_key_base"):
self._key_base = md5("%s%s%s" % (self.cffs.authurl, self.cffs.username, tenant_name)).hexdigest()
return "%s-%s" % (self._key_base, md5(smart_str(index)).hexdigest())
def __init__(self, key,secret,host,entity={}):
self.host = host
self.consumer_key = key
self.consumer_secret = secret
self.created = datetime.strptime(entity.get('created','2001-01-01T00:00:01'),'%Y-%m-%dT%H:%M:%S')
self.application= entity.get('application',None)
self.resource_uri= entity.get('resource_uri','')
self.id = int(entity.get('id','0'))
#TODO this self.key is over write app's consumer key
# I don't think self.key at object level is being use anywhere (only on application)
self.key = entity.get('key','')
self.original_key= entity.get('original_key','')
self.name = smart_str(entity.get('name',''), strings_only=True)
self.type = entity.get('type','')
self.views = entity.get('views',None)
self.shares = entity.get('shares',None)
self.likes = entity.get('likes',None)
self.comments = entity.get('comments',None)
self.meta = entity.get('meta',None)
try:
self.meta = simplejson.loads(self.meta)
except:
pass
self.total_activity = entity.get('total_activity',None)
def build_sign(self, **kwargs):
"""
build wechat pay signature
"""
sign_temp = '%s&key=%s' % (self.create_sign_string(**kwargs),
self.partnerKey)
return hashlib.md5(smart_str(sign_temp)).hexdigest().upper()
def set_params(self, **kwargs):
self.params = {}
for (k, v) in kwargs.items():
self.params[k] = smart_str(v)
self.params["nonce_str"] = random_str(32)
self.params["trade_type"] = self.trade_type
self.params.update(self.common_params)
def build_sign(self, **kwargs):
"""
build wechat pay signature
"""
sign_temp = '%s&key=%s' % (self.create_sign_string(**kwargs),
self.sign_key)
return hashlib.md5(smart_str(sign_temp)).hexdigest().upper()
def set_params(self, **kwargs):
self.params = {}
for (k, v) in kwargs.items():
self.params[k] = smart_str(v)
self.params["nonce_str"] = random_str(32)
if self.trade_type:
self.params["trade_type"] = self.trade_type
self.params.update(self.common_params)
def is_duplicate(self, table, row):
'''Checks to see if the row is in the database
Returns True if match found, else false
@table = the table you want to perform the test on
@row = the row whose existince your checking for
'''
# get names of all fields in the table except for the id
#fields = [x[1] for x in self.get_field_list(table)][1:]
# temporary fix to remove loc_id from logs table
#fields = [x for x in fields if x != 'loc_id']
# get table info (to be used when determining field type below)
#field_type = {x['name']:x['type'] for x in self._table_info(table)}
conditionals = []
for i, x in enumerate(row.items()):
field = smart_str(x[0])
value = smart_str(x[1])
if i == 0:
conditionals.append(u"WHERE {0}=\"{1}\"".format(
field, value.decode('utf8')))
else:
conditionals.append(u" AND {0}=\"{1}\"".format(
field, value.decode('utf8')))
conditionals = u"".join(x for x in conditionals)
field_names = tuple(row.keys())
field_values = u"VALUES{0}".format(
tuple([smart_str(x) for x in row.values()]))
sql = u"SELECT id FROM {0} {1}".format(table, conditionals)
out = u"INSERT INTO {0} {1} {2}".format(
table, field_names, field_values)
# if its not a duplicate, return the insert statment
rs = self.cursor.execute(sql).fetchone()
if not rs:
print(out)
return out # yield
else:
# its a duplicate
print('DUPLICATE')
return None
def wrapper(*args,**kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.debug("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except ClientException, e:
# some errno mapping
if e.http_status == 404:
err = ENOENT
elif e.http_status == 400:
err = EPERM
elif e.http_status == 403:
err = EACCES
else:
err = EIO
msg = "%s: %s" % (smart_str(e.msg), smart_str(e.http_reason))
log(msg)
raise IOSError(err, msg)
def wrapper(*args,**kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.debug("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except ClientException, e:
# some errno mapping
if e.http_status == 404:
err = ENOENT
elif e.http_status == 400:
err = EPERM
elif e.http_status == 403:
err = EACCES
else:
err = EIO
msg = "%s: %s" % (smart_str(e.msg), smart_str(e.http_reason))
log(msg)
raise IOSError(err, msg)
def tsina_post(content):
from local_config import post_url
content = content_filter(content)
data = { 'act': 'add', 'rl': '0', 'content': smart_str(content) }
resp = urlfetch.fetch(url=post_url,
payload=urlencode(data),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if resp.status_code != 200:
logging.warning('Failed posting to sina miniblog!')
def write(self, key, value, version=None):
key = utils.smart_str(key)
value = utils.smart_str(value)
with open(self.database, 'rb+') as f:
data = '%s\0%s' % (key, value)
length = len(data)
f.seek(0, 2)
position = f.tell()
if key in self.table:
previous, header = self.table[key]
if not self.overwrite and header.version != version:
raise Conflict(self.read(key), version)
header = Header(previous, header.version+1, length)
else:
header = Header(0, 0, length)
write_header(f, header)
f.write(data)
f.flush()
self.table[key] = (position, header)
return Document(header, key, value)
def make_regexp(q):
words = []
w = re.compile(r'\W+', re.U)
for k in q:
if k != 'operator':
words.extend(w.split(smart_str(q[k]).decode("utf-8")))
words = filter(lambda x: len(x) > 2, words)
words.sort(lambda x, y: len(y) - len(x))
patt = "|".join(words)
patt = "([\W|-]{1})(" + patt + ")([\W|-]{1})"
return re.compile(patt, re.I | re.U)
def stat(self, path):
"""
Returns an os.stat_result for path or raises IOSError.
Returns the information from the cache if possible.
"""
path = path.rstrip("/") or "/"
logging.debug("stat path %r" % (path))
directory, leaf = posixpath.split(path)
# Refresh the cache it if is old, or wrong
if not self.valid(directory):
logging.debug("invalid cache for %r (path: %r)" %
(directory, self.path))
self.listdir(directory)
if path != "/":
try:
stat_info = self.cache[smart_str(leaf)]
except KeyError:
logging.debug("Didn't find %r in directory listing" % leaf)
# it can be a container and the user doesn't have
# permissions to list the root
if directory == '/' and leaf:
try:
container = self.conn.head_container(leaf)
if (self.cffs.storage_policy is not None
and container['x-storage-policy'] !=
self.cffs.storage_policy):
raise IOSError(
ENOENT, 'No such file or directory %s' % leaf)
except ClientException:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
logging.debug(
"Accessing %r container without root listing" % leaf)
stat_info = self._make_stat(
count=int(container["x-container-object-count"]),
bytes=int(container["x-container-bytes-used"]),
)
else:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
else:
# Root directory size is sum of containers, count is containers
bytes = sum(stat_info.st_size for stat_info in self.cache.values())
count = len(self.cache)
stat_info = self._make_stat(count=count, bytes=bytes)
logging.debug("stat path: %r" % stat_info)
return stat_info
def __init__(self, key,secret,host,app_id, api_user):
'''
new app using app_dict = {}, id = 0
'''
self.host = host
self.consumer_key = key
self.consumer_secret = secret
self.app_id = app_id
if type(api_user)==int:
self.id = api_user
else:
self.id = int(api_user.get('id','0'))
self.resource_uri = api_user.get('resource_uri','')
self.created = datetime.strptime(api_user.get('created','2001-01-01T00:00:01'), '%Y-%m-%dT%H:%M:%S')
self.updated = datetime.strptime(api_user.get('updated','2000-01-01T00:00:01'),'%Y-%m-%dT%H:%M:%S')
self.username = smart_str(api_user.get('username',''), strings_only=True)
self.date_of_birth = api_user.get('date_of_birth','')
self.description = api_user.get('description','')
self.device_id = api_user.get('device_id','')
self.email = api_user.get('email','')
self.first_name = api_user.get('first_name','')
self.last_name = api_user.get('last_name','')
self.location = api_user.get('location','')
self.sex = api_user.get('sex','')
self.small_image_uri = api_user.get('small_image_uri','')
self.medium_image_uri = api_user.get('medium_image_uri','')
self.large_image_uri = api_user.get('large_image_uri','')
self.meta = api_user.get('meta',None)
self.stats = api_user.get('stats','{}')
self.user_devices = api_user.get('user_devices',[])
self.third_party_auth = api_user.get('third_party_auth',[])
self.reach = None
reach = 0
for tpa in self.third_party_auth:
if "connections_count" in tpa:
reach = reach + (tpa.get("connections_count",0) or 0)
if reach > 0:
self.reach = reach
def stat(self, path):
"""
Returns an os.stat_result for path or raises IOSError.
Returns the information from the cache if possible.
"""
path = path.rstrip("/") or "/"
logging.debug("stat path %r" % (path))
directory, leaf = posixpath.split(path)
# Refresh the cache it if is old, or wrong
if not self.valid(directory):
logging.debug("invalid cache for %r (path: %r)" % (directory, self.path))
self.listdir(directory)
if path != "/":
try:
stat_info = self.cache[smart_str(leaf)]
except KeyError:
logging.debug("Didn't find %r in directory listing" % leaf)
# it can be a container and the user doesn't have
# permissions to list the root
if directory == '/' and leaf:
try:
container = self.conn.head_container(leaf)
except ClientException:
raise IOSError(ENOENT, 'No such file or directory %s' % leaf)
logging.debug("Accessing %r container without root listing" % leaf)
stat_info = self._make_stat(count=int(container["x-container-object-count"]),
bytes=int(container["x-container-bytes-used"]),
)
else:
raise IOSError(ENOENT, 'No such file or directory %s' % leaf)
else:
# Root directory size is sum of containers, count is containers
bytes = sum(stat_info.st_size for stat_info in self.cache.values())
count = len(self.cache)
stat_info = self._make_stat(count=count, bytes=bytes)
logging.debug("stat path: %r" % stat_info)
return stat_info
def unserialize(js):
"""Unserialize a JSON object into a cache dict."""
return dict(((smart_str(key), os.stat_result(value)) for key, value in json.loads(js).iteritems()))
def listdir_container(self, cache, container, path=""):
"""Fills cache with the list dir of the container"""
container = smart_str(container)
path = smart_str(path)
logging.debug("listdir container %r path %r" % (container, path))
if path:
prefix = path.rstrip("/")+"/"
else:
prefix = None
_, objects = self.conn.get_container(container, prefix=prefix, delimiter="/")
# override 10000 objects limit with markers
nbobjects = len(objects)
while nbobjects >= 10000:
# get last object as a marker
lastobject = objects[-1]
if 'subdir' in lastobject:
# {u'subdir': 'dirname'}
lastobjectname = lastobject['subdir'].rstrip("/")
else:
lastobjectname = lastobject['name']
# get a new list with the marker
_, newobjects = self.conn.get_container(container, prefix=prefix, delimiter="/", marker=lastobjectname)
# get the new list length
nbobjects = len(newobjects)
logging.debug("number of objects after marker %s: %s" % (lastobjectname, nbobjects))
# add the new list to current list
objects.extend(newobjects)
logging.debug("total number of objects %s:" % len(objects))
if self.cffs.hide_part_dir:
manifests = {}
for obj in objects:
# {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'},
if 'subdir' in obj:
# {u'subdir': 'dirname'}
obj['name'] = obj['subdir'].rstrip("/")
# If a manifest and it's segment directory have the
# same name then we have to choose which we want to
# show, we can't show both. So we choose to keep the
# manifest if hide_part_dir is enabled.
#
# We can do this here because swift returns objects in
# alphabetical order so the manifest will come before
# its segments.
if self.cffs.hide_part_dir and obj['name'] in manifests:
logging.debug("Not adding subdir %s which would overwrite manifest" % obj['name'])
continue
elif obj.get('bytes') == 0 and obj.get('hash') and obj.get('content_type') != 'application/directory':
# if it's a 0 byte file, has a hash and is not a directory, we make an extra call
# to check if it's a manifest file and retrieve the real size / hash
manifest_obj = self.conn.head_object(container, obj['name'])
logging.debug("possible manifest file: %r" % manifest_obj)
if 'x-object-manifest' in manifest_obj:
if self.cffs.hide_part_dir:
manifests[obj['name']] = unicode(unquote(manifest_obj['x-object-manifest']), "utf-8")
logging.debug("manifest found: %s" % manifest_obj['x-object-manifest'])
obj['hash'] = manifest_obj['etag']
obj['bytes'] = int(manifest_obj['content-length'])
obj['count'] = 1
# Keep all names in utf-8, just like the filesystem
name = posixpath.basename(obj['name']).encode("utf-8")
cache[name] = self._make_stat(**obj)
if self.cffs.hide_part_dir:
for manifest in manifests:
manifest_container, manifest_obj = parse_fspath('/' + manifests[manifest])
if manifest_container == container:
for cache_obj in cache.copy():
# hide any manifest segments, but not the manifest itself, if it
# happens to share a prefix with its segments.
if unicode(unquote(cache_obj), "utf-8") != manifest and \
unicode(unquote(os.path.join(path, cache_obj)), "utf-8").startswith(manifest_obj):
logging.debug("hiding manifest %r segment %r" % (manifest, cache_obj))
del cache[cache_obj]
def __str__(self):
return smart_str("Comment Object: %s" % self.id)
def __str__(self):
return smart_str("Evis Object: %s (%s)" % (self.id, self.evi_permalink))
def __str__(self):
return smart_str("File Object: %s (%s)" % (self.id, self.fle_permalink))
def __str__(self):
return smart_str("Node Object: %s (%s)" % (self.id, self.nod_name))
def __str__(self):
return smart_str("Member Object: %s (%s)" % (self.id, str(self.mem_name)))
def __repr__(self):
try:
u = unicode(self)
except (UnicodeEncodeError, UnicodeDecodeError):
u = '[Bad Unicode data]'
return smart_str(u'<{}: {}>'.format(self.__class__.__name__, u))
def unserialize(js):
"""Unserialize a JSON object into a cache dict."""
return dict(((smart_str(key), os.stat_result(value)) for key, value in json.loads(js).iteritems()))
def __init__(self, key,secret,host,app={}):
''' if app = int return app(id)
elif app = {} init app with dict
new app using app = {}, id = 0
'''
self.host = host
self.consumer_key = key
self.consumer_secret = secret
if type(app)==int:
self.id = app
self.refresh()
else:
## can't modify
self.id =int(app.get('id',0))
self.created =app.get('created','')
self.deleted =app.get('deleted','')
self.last_saved =app.get('last_saved','')
self.socialize_consumer_key =app.get('socialize_consumer_key','')
self.socialize_consumer_secret =app.get('socialize_consumer_secret','')
self.socialize_app =app.get('socialize_app','')
self.push_certificate =app.get('push_certificate', None)
self.is_socialize_editable =app.get('is_socialize_editable', True)
self.c2dm_token_source =app.get('c2dm_token_source', 'socialize')
self.custom_propagation_domain =app.get('custom_propagation_domain', None)
self.twitter_consumer_key =app.get('twitter_consumer_key', None)
self.twitter_consumer_secret =app.get('twitter_consumer_secret', None)
self.notification_quotas =app.get('quotas', {})
self.__update_notification_quotas()
## modifiable
notifications_enabled =app.get('notifications_enabled', False)
if type(notifications_enabled) == str:
if notifications_enabled == "false":
notifications_enabled = False
else:
notifications_enabled = True
self.notifications_enabled =notifications_enabled
self.android_package_name =app.get('android_package_name','')
self.c2dm_sender_auth_token =app.get('c2dm_sender_auth_token','')
self.apple_store_id =app.get('apple_store_id','')
self.category =app.get('category','')
self.description =smart_str(app.get('description',''), strings_only=True)
self.name =smart_str(app.get('name',''), strings_only=True)
self.mobile_platform =app.get('platforms',[])
self.resource_uri =app.get('resource_uri','')
self.stats =app.get('stats','')
self.display_name =self.name
self.icon_url =app.get('icon_url',None)
self.stats =app.get('stats',{})
if len(self.stats) > 0:
self.__calculate_stats(self.stats )
#logger.info(self.stats)
#to make forward and backward compatible with API-user changes (temporary)
user_id = int(app.get('user','0'))
if user_id == 0:
user_id = int(app.get('user_id','0'))
self.user =user_id
def listdir_container(self, cache, container, path=""):
"""Fills cache with the list dir of the container"""
container = smart_str(container)
path = smart_str(path)
logging.debug("listdir container %r path %r" % (container, path))
if path:
prefix = path.rstrip("/")+"/"
else:
prefix = None
_, objects = self.conn.get_container(container, prefix=prefix, delimiter="/")
# override 10000 objects limit with markers
nbobjects = len(objects)
while nbobjects >= 10000:
# get last object as a marker
lastobject = objects[-1]
if 'subdir' in lastobject:
# {u'subdir': 'dirname'}
lastobjectname = lastobject['subdir'].rstrip("/")
else:
lastobjectname = lastobject['name']
# get a new list with the marker
_, newobjects = self.conn.get_container(container, prefix=prefix, delimiter="/", marker=lastobjectname)
# get the new list length
nbobjects = len(newobjects)
logging.debug("number of objects after marker %s: %s" % (lastobjectname, nbobjects))
# add the new list to current list
objects.extend(newobjects)
logging.debug("total number of objects %s:" % len(objects))
if self.cffs.hide_part_dir:
manifests = []
for obj in objects:
# {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'},
if 'subdir' in obj:
# {u'subdir': 'dirname'}
obj['name'] = obj['subdir'].rstrip("/")
elif obj.get('bytes') == 0 and obj.get('hash') and obj.get('content_type') != 'application/directory':
# if it's a 0 byte file, has a hash and is not a directory, we make an extra call
# to check if it's a manifest file and retrieve the real size / hash
manifest_obj = self.conn.head_object(container, obj['name'])
logging.debug("possible manifest file: %r" % manifest_obj)
if 'x-object-manifest' in manifest_obj:
if self.cffs.hide_part_dir:
manifests.append(manifest_obj['x-object-manifest'])
logging.debug("manifest found: %s" % manifest_obj['x-object-manifest'])
obj['hash'] = manifest_obj['etag']
obj['bytes'] = int(manifest_obj['content-length'])
obj['count'] = 1
# Keep all names in utf-8, just like the filesystem
name = posixpath.basename(obj['name']).encode("utf-8")
cache[name] = self._make_stat(**obj)
if self.cffs.hide_part_dir:
for manifest in manifests:
manifest_container, manifest_obj = parse_fspath('/' + manifest)
if manifest_container == container:
for cache_obj in cache.copy():
if manifest_obj == os.path.join(path, cache_obj):
logging.debug("hidding part dir %r" % manifest_obj)
del cache[cache_obj]
def listdir_container(self, cache, container, path=""):
"""Fills cache with the list dir of the container"""
container = smart_str(container)
path = smart_str(path)
logging.debug("listdir container %r path %r" % (container, path))
if path:
prefix = path.rstrip("/") + "/"
else:
prefix = None
_, objects = self.conn.get_container(container,
prefix=prefix,
delimiter="/")
# override 10000 objects limit with markers
nbobjects = len(objects)
while nbobjects >= 10000:
# get last object as a marker
lastobject = objects[-1]
if 'subdir' in lastobject:
# {u'subdir': 'dirname'}
lastobjectname = lastobject['subdir'].rstrip("/")
else:
lastobjectname = lastobject['name']
# get a new list with the marker
_, newobjects = self.conn.get_container(container,
prefix=prefix,
delimiter="/",
marker=lastobjectname)
# get the new list length
nbobjects = len(newobjects)
logging.debug("number of objects after marker %s: %s" %
(lastobjectname, nbobjects))
# add the new list to current list
objects.extend(newobjects)
logging.debug("total number of objects %s:" % len(objects))
if self.cffs.hide_part_dir:
manifests = {}
for obj in objects:
# {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'},
if 'subdir' in obj:
# {u'subdir': 'dirname'}
obj['name'] = obj['subdir'].rstrip("/")
# If a manifest and it's segment directory have the
# same name then we have to choose which we want to
# show, we can't show both. So we choose to keep the
# manifest if hide_part_dir is enabled.
#
# We can do this here because swift returns objects in
# alphabetical order so the manifest will come before
# its segments.
if self.cffs.hide_part_dir and obj['name'] in manifests:
logging.debug(
"Not adding subdir %s which would overwrite manifest" %
obj['name'])
continue
elif obj.get('bytes') == 0 and obj.get('hash') and obj.get(
'content_type') != 'application/directory':
# if it's a 0 byte file, has a hash and is not a directory, we make an extra call
# to check if it's a manifest file and retrieve the real size / hash
manifest_obj = self.conn.head_object(container, obj['name'])
logging.debug("possible manifest file: %r" % manifest_obj)
if 'x-object-manifest' in manifest_obj:
if self.cffs.hide_part_dir:
manifests[obj['name']] = smart_unicode(
unquote(manifest_obj['x-object-manifest']),
"utf-8")
logging.debug("manifest found: %s" %
manifest_obj['x-object-manifest'])
obj['hash'] = manifest_obj['etag']
obj['bytes'] = int(manifest_obj['content-length'])
obj['count'] = 1
# Keep all names in utf-8, just like the filesystem
name = posixpath.basename(obj['name']).encode("utf-8")
cache[name] = self._make_stat(**obj)
if self.cffs.hide_part_dir:
for manifest in manifests:
manifest_container, manifest_obj = parse_fspath(
'/' + manifests[manifest])
if manifest_container == container:
for cache_obj in cache.copy():
# hide any manifest segments, but not the manifest itself, if it
# happens to share a prefix with its segments.
if unicode(unquote(cache_obj), "utf-8") != manifest and \
unicode(unquote(os.path.join(path, cache_obj)), "utf-8").startswith(manifest_obj):
logging.debug("hiding manifest %r segment %r" %
(manifest, cache_obj))
del cache[cache_obj]
