def setcontents(self, path, data, chunk_size=1024 * 64):
if not isinstance(data, six.binary_type):
return super(MemoryFS, self).setcontents(path, data, chunk_size)
if not self.exists(path):
self.open(path, 'wb').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
raise ResourceInvalidError('Not a directory %(path)s', path)
new_mem_file = StringIO()
new_mem_file.write(data)
dir_entry.mem_file = new_mem_file
def setcontents(self, path, data, chunk_size=1024*64):
if not isinstance(data, str):
return super(MemoryFS, self).setcontents(path, data, chunk_size)
if not self.exists(path):
self.open(path, 'w').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
raise ResourceInvalidError('Not a directory %(path)s', path)
new_mem_file = StringIO()
new_mem_file.write(data)
dir_entry.mem_file = new_mem_file
def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=1024*64):
if isinstance(data, six.binary_type):
if not self.exists(path):
self.open(path, 'wb').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
raise ResourceInvalidError('Not a directory %(path)s', path)
new_mem_file = StringIO()
new_mem_file.write(data)
dir_entry.mem_file = new_mem_file
return len(data)
return super(MemoryFS, self).setcontents(path, data=data, encoding=encoding, errors=errors, chunk_size=chunk_size)
def setcontents(self,
path,
data=b'',
encoding=None,
errors=None,
chunk_size=1024 * 64,
**kwargs):
if isinstance(data, six.binary_type):
if not self.exists(path):
self.open(path, 'wb').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
raise ResourceInvalidError('Not a directory %(path)s', path)
new_mem_file = StringIO()
new_mem_file.write(data)
dir_entry.mem_file = new_mem_file
return len(data)
return super(MemoryFS, self).setcontents(path,
data=data,
encoding=encoding,
errors=errors,
chunk_size=chunk_size,
**kwargs)
class DirEntry(object):
def sync(f):
def deco(self, *args, **kwargs):
if self.lock is not None:
try:
self.lock.acquire()
return f(self, *args, **kwargs)
finally:
self.lock.release()
else:
return f(self, *args, **kwargs)
return deco
def __init__(self, type, name, contents=None):
assert type in ("dir", "file"), "Type must be dir or file!"
self.type = type
self.name = name
if contents is None and type == "dir":
contents = {}
self.open_files = []
self.contents = contents
self.mem_file = None
self.created_time = datetime.datetime.now()
self.modified_time = self.created_time
self.accessed_time = self.created_time
self.xattrs = {}
self.lock = None
if self.type == 'file':
self.mem_file = StringIO()
self.lock = threading.RLock()
def get_value(self):
self.lock.acquire()
try:
return self.mem_file.getvalue()
finally:
self.lock.release()
data = property(get_value)
def desc_contents(self):
if self.isfile():
return "<file %s>" % self.name
elif self.isdir():
return "<dir %s>" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems())
def isdir(self):
return self.type == "dir"
def isfile(self):
return self.type == "file"
def __str__(self):
return "%s: %s" % (self.name, self.desc_contents())
@sync
def __getstate__(self):
state = self.__dict__.copy()
state.pop('lock')
if self.mem_file is not None:
state['mem_file'] = self.data
return state
def __setstate__(self, state):
self.__dict__.update(state)
if self.type == 'file':
self.lock = threading.RLock()
else:
self.lock = None
if self.mem_file is not None:
data = self.mem_file
self.mem_file = StringIO()
self.mem_file.write(data)
class DirEntry(object):
def sync(f):
def deco(self, *args, **kwargs):
if self.lock is not None:
try:
self.lock.acquire()
return f(self, *args, **kwargs)
finally:
self.lock.release()
else:
return f(self, *args, **kwargs)
return deco
def __init__(self, type, name, contents=None):
assert type in ("dir", "file"), "Type must be dir or file!"
self.type = type
self.name = name
if contents is None and type == "dir":
contents = {}
self.open_files = []
self.contents = contents
self.mem_file = None
self.created_time = datetime.datetime.now()
self.modified_time = self.created_time
self.accessed_time = self.created_time
self.xattrs = {}
self.lock = None
if self.type == 'file':
self.mem_file = StringIO()
self.lock = threading.RLock()
def get_value(self):
self.lock.acquire()
try:
return self.mem_file.getvalue()
finally:
self.lock.release()
data = property(get_value)
def desc_contents(self):
if self.isfile():
return "<file %s>" % self.name
elif self.isdir():
return "<dir %s>" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems())
def isdir(self):
return self.type == "dir"
def isfile(self):
return self.type == "file"
def __str__(self):
return "%s: %s" % (self.name, self.desc_contents())
@sync
def __getstate__(self):
state = self.__dict__.copy()
state.pop('lock')
if self.mem_file is not None:
state['mem_file'] = self.data
return state
def __setstate__(self, state):
self.__dict__.update(state)
if self.type == 'file':
self.lock = threading.RLock()
else:
self.lock = None
if self.mem_file is not None:
data = self.mem_file
self.mem_file = StringIO()
self.mem_file.write(data)
class RiakFSObject(DirEntry):
"""
Represents a filesystem "node", either a directory of file.
A directory node may have sub-nodes in a contents dictionary.
Has more responsibility than the `DirEntry` class, for example the
`remove` and `_make_dir_entry` methods. Also has a `path` attribute
which, in the case of a file, is the key of the object in the Riak
store. TODO: look into moving the responsibilty back to the FS object,
make this class dumber.
"""
@classmethod
def from_dict(cls, bucket, data):
def obj_from_dict(d):
type = d.pop("type")
name = d.pop("name")
prefix = d.pop("prefix", None)
contents = d.pop("contents", {})
obj = cls(bucket, type, name, prefix)
obj.xattrs = d["xattrs"]
obj.timestamps = d["timestamps"]
for k, v in contents.items():
obj.contents[k] = obj_from_dict(v)
return obj
return obj_from_dict(data)
def to_dict(self):
ignore = set(["bucket", "contents", "lock", "open_files"])
def serialize(obj):
d = {}
for k, v in obj.__dict__.iteritems():
if k in ignore or k.startswith("_"):
continue
if k == "xattrs" and not v:
continue
d[k] = v
if obj.contents:
d["contents"] = dict((k, serialize(v)) for k, v in obj.contents.items())
return d
return serialize(self)
# Datetime instances don't json serialize, so we maintain them as lists
# under the hood and "rehydrate" on demand
def _get_ct(self):
return datetime.fromtimestamp(time.mktime(self.timestamps["ctime"]))
def _get_mt(self):
return datetime.fromtimestamp(time.mktime(self.timestamps["mtime"]))
def _get_at(self):
return datetime.fromtimestamp(time.mktime(self.timestamps["atime"]))
def _set_ct(self, val):
self.timestamps["ctime"] = list(val.timetuple())
def _set_mt(self, val):
self.timestamps["mtime"] = list(val.timetuple())
def _set_at(self, val):
self.timestamps["atime"] = list(val.timetuple())
created_time = property(_get_ct, _set_ct)
modified_time = property(_get_mt, _set_mt)
accessed_time = property(_get_at, _set_at)
def _get_file(self):
if self.type == "file" and self._mem_file is None:
bytes = self.bucket.get_binary(self.path).get_data()
self._mem_file = StringIO(bytes)
return self._mem_file
def _set_file(self, stream):
self._mem_file = stream
mem_file = property(_get_file, _set_file)
def __init__(self, bucket, type, name, prefix=None, contents=None):
assert type in ("dir", "file"), "Type must be dir or file!"
self.bucket = bucket
self.type = type
self.name = name.rstrip("/")
if prefix:
prefix = prefix.strip("/") + "/"
else:
prefix = ""
self.prefix = prefix
self.path = prefix + name
if type == "dir":
self.path += "/"
if contents is None:
contents = {}
self.open_files = []
self.contents = contents
now = list(datetime.now().timetuple())
self.timestamps = {"ctime": now, "mtime": now, "atime": now}
self.xattrs = {}
self.lock = None
self._mem_file = None
if self.type == "file":
self.lock = threading.RLock()
# problem of having a `path` attribute - if there are contents, their
# paths may be different.
# if contents:
# def update_paths(entries, prefix):
# prefix = '/' + prefix.strip('/') + '/'
# for entry in entries:
# entry.prefix = prefix
# entry.path = prefix + entry.name
# if entry.contents:
# update_paths(entry.contents.values(), entry.path)
# update_paths(contents.values(), self.path)
def _make_dir_entry(self, type, name, contents=None):
child = self.__class__(self.bucket, type, name, prefix=self.path, contents=contents)
self.contents[name] = child
return child
def remove(self, name):
entry = self.contents[name]
if entry.isfile():
key = self.path + name
obj = self.bucket.get(key)
obj.delete()
else:
for child in entry.contents.keys():
entry.remove(child)
del self.contents[name]
def __getstate__(self):
state = self.__dict__.copy()
del state["lock"]
bucket = state.pop("bucket")
state["bucket"] = bucket.get_name()
state["host"] = bucket._client._host
state["port"] = bucket._client._port
state["transport"] = bucket._client._transport.__class__.__name__[4:-9].upper()
if self._mem_file is not None:
state["_mem_file"] = self.data
return state
def __setstate__(self, state):
state["bucket"] = RiakBucket(state.pop("bucket"), state.pop("host"), state.pop("port"), state.pop("transport"))
self.__dict__.update(state)
if self.type == "file":
self.lock = threading.RLock()
else:
self.lock = None
if self._mem_file is not None:
data = self._mem_file
self._mem_file = StringIO()
self._mem_file.write(data)
