def _send(self, src, dest):
# Dropbox limitation - the final file name must match the source
# name. AND we cannot pass the final file name in.
old_src = None
dest_folder, dest_fname = os.path.split(dest)
src_folder, src_fname = os.path.split(src)
if dest_fname != src_fname:
# We need to rename src temporarily
# because the src name and final name MUST match
# We do this in a temporary folder beside its current location.
old_src = src
tmpfolder = tempfile.mkdtemp()
src = os.path.join(tmpfolder, dest_fname)
os.rename(old_src, src)
log.debug("Fixed name", old_src, src)
try:
dest = utils.join_paths(self.root, dest_folder)
with open(src, "rb") as fd:
log.debug("Dropbox Sending", src)
ret = self.db_client.put_file(self.db_root, dest, fd)
log.debug("Dropbox Send complete.", ret)
if ret.status != 200:
log.debug("Dropbox send failed")
raise IOError("Unable to send file: " + ret.reason)
finally:
if old_src:
# Put the name back!
os.rename(src, old_src)
os.rmdir(tmpfolder)
def _make_dir(self, folder):
"""
Create a folder on thel FTP service.
If its relative, the folder is made relative to cwd.
Otherwise its absolute.
We do it by attempting to create every folder in the chain.
"""
folder = utils.join_paths(self.root, folder)
self._pushd(".")
try:
# Get all the anscestor paths.
paths = utils.ancestor_paths(folder)
for path in paths:
if not path in ["", ".", "..", "/"]:
try:
# If we can't chdir, then we try to build
self._pushd(path)
self._popd()
except:
# If we fail to build - we fail
if self.sftp and use_paramiko:
self.ftp.mkdir(path)
else:
self.ftp.mkd(path)
finally:
self._popd()
def _remove_file(self, path):
log.info("Removing file", path)
path = utils.join_paths(self.root, path)
if not os.path.isfile(path):
raise Exception(_("Missing file"))
os.remove(path)
if os.path.isfile(path):
raise Exception("Unable to delete file")
def _list(self, dir):
if dir in ["", ".", "/"]:
path = self.root
else:
path = utils.join_paths(self.root, dir)
resp = self.db_client.metadata(self.db_root, path, file_limit=10000, list="true")
ret = [os.path.basename(item["path"]) for item in resp.data["contents"]]
return ret
def _size(self, path):
path = utils.join_paths(self.root, path)
if self.sftp and use_paramiko:
stat = self.ftp.stat(path)
size = stat.st_size
else:
self.ftp.sendcmd("TYPE i")
size = self.ftp.size(path)
return size
def _make_dir(self, folder):
# Make the folder. utils.makedirs wont fail if the folder exists.
if folder in ["", ".", "/"]:
folder = self.root
else:
folder = utils.join_paths(self.root, folder)
utils.makedirs(folder)
if not os.path.isdir(folder):
raise Exception("Unable to build folder")
def _remove_dir(self, path):
log.info("Removing dir", path)
if path in ["", "."]:
path = self.root
else:
path = utils.join_paths(self.root, path)
if not os.path.isdir(path):
raise Exception("Missing folder")
shutil.rmtree(path)
if os.path.isdir(path):
raise Exception("Unable to delete folder")
def _list(self, path="."):
path = utils.join_paths(self.root, path)
# Save the folder
self._pushd(path)
try:
if self.sftp and use_paramiko:
contents = self.ftp.listdir()
else:
contents = self.ftp.nlst()
finally:
self._popd() # Restore the working folder
return contents
def open(self, path, mode):
if not self.connected:
self.connect()
self.make_dir(os.path.split(path)[0])
self.io_path = utils.join_paths(self.root, path)
if "w" in mode:
self.io_state = ioWriting
self.io_fd = open(self.io_path, "wb")
else:
self.io_state = ioReading
self.io_fd = open(self.io_path, "rb")
return self
def _get(self, src, dest):
src = utils.join_paths(self.root, src)
with open(dest, "wb") as fd:
fileobj = self.db_client.get_file(self.db_root, src)
if fileobj.status != 200:
raise IOError("Unable to get %s: %s" % (src, fileobj.reason))
try:
data = fileobj.read(const.BufferSize)
while len(data) > 0:
fd.write(data)
data = fileobj.read(const.BufferSize)
finally:
fileobj.close()
def _send(self, src, dest):
dest = utils.join_paths(self.root, dest)
with open(src, "rb") as ifd:
if self.sftp and use_paramiko:
log.info("Starting sftp.paramiko put call src", src, "dest", dest)
ofd = self.ftp.open(dest, "w+")
try:
data = ifd.read(const.BufferSize)
while data:
log.debug("FTP read %d bytes", len(data))
ofd.write(data)
data = ifd.read(const.BufferSize)
finally:
ofd.close()
else:
self.ftp.storbinary("STOR " + dest, ifd)
def _get(self, src, dest):
src = utils.join_paths(self.root, src)
with open(dest, "wb") as ofd:
if self.sftp and use_paramiko:
ifd = self.ftp.open(src, "r")
try:
data = ifd.read(const.BufferSize)
while data:
log.debug("FTP read %d bytes", len(data))
ofd.write(data)
data = ifd.read(const.BufferSize)
finally:
ifd.close()
else:
self.ftp.retrbinary("RETR " + src, ofd.write)
def _remove_file(self, path):
if path in ["", ".", "/"]:
path = self.root
else:
path = utils.join_paths(self.root, path)
self.db_client.file_delete(self.db_root, path)
def _get(self, src, dest):
src = utils.join_paths(self.root, src)
shutil.copyfile(src, dest)
return dest
def _send(self, src, dest):
# Switch to absolute path.
rem_dest = utils.join_paths(self.root, dest)
shutil.copyfile(src, rem_dest)
return dest
def _remove_file(self, path):
path = utils.join_paths(self.root, path)
if self.sftp and use_paramiko:
self.ftp.remove(path)
else:
self.ftp.delete(path)
def _size(self, path):
path = utils.join_paths(self.root, path)
return os.path.getsize(path)
def _list(self, dir):
path = utils.join_paths(self.root, dir)
return os.listdir(path)
def _remove_dir(self, path):
if path in ["", ".", "/"]:
path = self.root
else:
path = utils.join_paths(self.root, path)
self._recurse_delete(path)
def _make_dir(self, folder):
if folder in ["", "."]:
path = self.root
else:
path = utils.join_paths(self.root, folder)
self.db_client.file_create_folder(self.db_root, path)
def _size(self, path):
path = utils.join_paths(self.root, path)
resp = self.db_client.metadata(self.db_root, path, list="false")
return resp.data["bytes"]