def run(self):
try:
self.preload()
source_abs_path = os.path.abspath(self.source_path)
target_abs_path = os.path.abspath(self.target_path)
self.logger.debug(
"FM WebDav NewFile worker run(), source_abs_path = %s, target_abs_path = %s"
% (source_abs_path, target_abs_path))
webdav_connection = WebDavConnection.create(
self.login, self.session.get('server_id'), self.logger)
try:
if webdav_connection.isdir(source_abs_path):
source_abs_path += '/'
target_abs_path += '/'
source_info = webdav_connection.generate_file_info(
source_abs_path)
webdav_connection.move_file(source_abs_path, target_abs_path)
target_info = webdav_connection.generate_file_info(
target_abs_path)
webdav_result = {"source": source_info, "target": target_info}
result = {
"data": webdav_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = WebDavConnection.get_error(
e, "Unable to rename source element.")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def get_total(self,
progress_object,
paths,
count_dirs=True,
count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" %
(count_dirs, count_files))
webdav = WebDavConnection.create(self.login,
self.target.get('server_id'),
self.logger)
for path in paths:
try:
if count_dirs:
progress_object["total"] += 1
for file in webdav.listdir(path):
if webdav.isdir(file):
progress_object["total"] += 1
else:
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" %
(str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug(
"FM WebDav ListFiles worker run(), abs_path = %s" % abs_path)
webdav_connection = WebDavConnection.create(
self.login, self.session.get('server_id'), self.logger)
listing = webdav_connection.list(path=abs_path)
result = {
"data": listing,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM WebDav ListFiles worker run(), abs_path = %s" % abs_path)
webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
listing = webdav_connection.list(path=abs_path)
result = {
"data": listing,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = self.get_abs_path(self.path)
self.logger.debug(
"FM ReadRulesWebDav worker run(), abs_path = %s" % abs_path)
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
htaccess_path = os.path.join(self.path, '.htaccess')
if not webdav.exists(htaccess_path):
default_rules = {
'allow_all': True,
'deny_all': False,
'order': 'Allow,Deny',
'denied': [],
'allowed': []
}
result = {
"data": default_rules,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
return
with webdav.open(abs_path) as fd:
content = fd.read()
htaccess = HtAccess(content, self.logger)
answer = htaccess.parse_file_content()
answer['allowed'] = htaccess.get_htaccess_allowed_ip()
answer['denied'] = htaccess.get_htaccess_denied_ip()
result = {
"data": answer,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def __init__(self, paths, session, *args, **kwargs):
super(ReadImages, self).__init__(*args, **kwargs)
self.paths = paths
self.session = session
self.webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
def run(self):
try:
self.preload()
source_abs_path = os.path.abspath(self.source_path)
target_abs_path = os.path.abspath(self.target_path)
self.logger.debug("FM WebDav NewFile worker run(), source_abs_path = %s, target_abs_path = %s" %
(source_abs_path, target_abs_path))
webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
try:
if webdav_connection.isdir(source_abs_path):
source_abs_path += '/'
target_abs_path += '/'
source_info = webdav_connection.generate_file_info(source_abs_path)
webdav_connection.move_file(source_abs_path, target_abs_path)
target_info = webdav_connection.generate_file_info(target_abs_path)
webdav_result = {
"source": source_info,
"target": target_info
}
result = {
"data": webdav_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = WebDavConnection.get_error(e, "Unable to rename source element.")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def __init__(self, path, session, *args, **kwargs):
super(ReadFile, self).__init__(*args, **kwargs)
self.path = path
# чем больше - тем точнее определяется кодировка, но медленнее, 50000 - выбрано опытным путем
self.charset_detect_buffer = 50000
self.session = session
self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
def __init__(self, path, encoding, session, *args, **kwargs):
super(ReadFile, self).__init__(*args, **kwargs)
self.path = path
self.encoding = encoding
# чем больше - тем точнее определяется кодировка, но медленнее, 50000 - выбрано опытным путем
self.charset_detect_buffer = 50000
self.session = session
self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
def run(self):
try:
self.preload()
self.logger.info("WebDav UploadFile process run")
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
target_file = "{0}{1}".format(self.path,
os.path.basename(self.file_path))
if not webdav.exists(target_file):
upload_result = webdav.upload(self.file_path, self.path)
if not upload_result['success'] or len(
upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception("Upload error")
os.remove(self.file_path)
elif self.overwrite and webdav.exists(
target_file) and not webdav.isdir(target_file):
upload_result = webdav.upload(self.file_path, self.path)
if not upload_result['success'] or len(
upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception("Upload error")
os.remove(self.file_path)
elif self.overwrite and webdav.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
webdav.remove(target_file)
upload_result = webdav.upload(self.file_path, self.path)
if not upload_result['success'] or len(
upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception("Upload error")
os.remove(self.file_path)
else:
pass
result = {"success": True}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM WebDav MakeDir worker run(), abs_path = %s" %
abs_path)
webdav_connection = WebDavConnection.create(
self.login, self.session.get('server_id'), self.logger)
try:
webdav_connection.mkdir(abs_path)
info = webdav_connection.info(abs_path)
fileinfo = {
"name": abs_path,
"mode": webdav_connection.getmode(info),
"mtime": str(info['modified'])
}
result = {
"data": fileinfo,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = WebDavConnection.get_error(
e, "File path already exists")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def get_total(self, progress_object, paths, count_files=True):
self.logger.debug("start get_total() files = %s" % count_files)
webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
for path in paths:
try:
self.recursive_total(webdav, path, progress_object)
except Exception as e:
self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total(), found %s objects" % progress_object.get("total"))
return
def get_total(self, progress_object, paths, count_files=True):
self.logger.debug("start get_total() files = %s" % count_files)
webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
for path in paths:
try:
self.recursive_total(webdav, path, progress_object)
except Exception as e:
self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total(), found %s objects" % progress_object.get("total"))
return
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM WebDav MakeDir worker run(), abs_path = %s" % abs_path)
webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
try:
webdav_connection.mkdir(abs_path)
info = webdav_connection.info(abs_path)
fileinfo = {
"name": abs_path,
"mode": webdav_connection.getmode(info),
"mtime": str(info['modified'])
}
result = {
"data": fileinfo,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = WebDavConnection.get_error(e, "File path already exists")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def __init__(self, source, target, paths, overwrite, *args, **kwargs):
super(MoveFromWebDav, self).__init__(*args, **kwargs)
self.source = source
self.target = target
self.paths = paths
self.overwrite = overwrite
self.webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
self.operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0,
"previous_percent": 0
}
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
next_tick = time.time() + REQUEST_DELAY
webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
for path in self.paths:
try:
webdav.remove(path)
success_paths.append(path)
if time.time() > next_tick:
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
self.on_running(self.status_id, progress=progress, pid=self.pid, pname=self.name)
next_tick = time.time() + REQUEST_DELAY
except Exception as e:
self.logger.error("Error removing file %s , error %s" % (str(path), str(e)))
error_paths.append(path)
result = {
"success": success_paths,
"errors": error_paths
}
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def __init__(self, source, target, paths, overwrite, *args, **kwargs):
super(MoveFromWebDav, self).__init__(*args, **kwargs)
self.source = source
self.target = target
self.paths = paths
self.overwrite = overwrite
self.webdav = WebDavConnection.create(self.login,
self.source.get('server_id'),
self.logger)
self.operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0,
"previous_percent": 0
}
def run(self):
try:
self.preload()
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" %
abs_path)
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
htaccess_path = os.path.join(self.path, '.htaccess')
if not webdav.exists(htaccess_path):
fd = webdav.open(htaccess_path, 'w')
fd.close()
with webdav.open(htaccess_path, 'r') as fd:
old_content = fd.read()
htaccess = HtAccess(old_content, self.logger)
content = htaccess.write_htaccess_file(self.params)
with webdav.open(htaccess_path, 'w') as fd:
fd.write(content)
result = {
"data": self.params,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def get_total(self, progress_object, paths, count_dirs=True, count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" % (count_dirs, count_files))
webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
for path in paths:
try:
if count_dirs:
progress_object["total"] += 1
for file in webdav.listdir(path):
if webdav.isdir(file):
progress_object["total"] += 1
else:
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def make_listing(self):
if self.session_type == Module.HOME:
path = self.path if self.path is not None else self.get_home_dir()
abs_path = self.get_abs_path(path)
items = []
self.__list_recursive(abs_path, items, 1)
result = {
'path': self.get_rel_path(abs_path),
'items': items
}
return result
if self.session_type == Module.FTP:
self.logger.info("FTP Listing preload")
path = self.path if self.path is not None else '/'
abs_path = os.path.abspath(path)
ftp_connection = self.get_ftp_connection(self.session)
listing = ftp_connection.list(path=abs_path)
return listing
if self.session_type == Module.SFTP:
self.logger.info("SFTP Listing preload")
path = self.path if self.path is not None else '.'
sftp_connection = self.get_sftp_connection(self.session)
listing = sftp_connection.list(path=path)
return listing
if self.session_type == Module.WEBDAV:
self.logger.info("WebDav Listing preload")
path = self.path if self.path is not None else '/'
abs_path = os.path.abspath(path)
webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
listing = webdav_connection.list(path=abs_path)
return listing
raise Exception("Unknown session type")
def run(self):
try:
self.preload()
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" % abs_path)
webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
htaccess_path = os.path.join(self.path, '.htaccess')
if not webdav.exists(htaccess_path):
fd = webdav.open(htaccess_path, 'w')
fd.close()
with webdav.open(htaccess_path, 'r') as fd:
old_content = fd.read()
htaccess = HtAccess(old_content, self.logger)
content = htaccess.write_htaccess_file(self.params)
with webdav.open(htaccess_path, 'w') as fd:
fd.write(content)
result = {
"data": self.params,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info(
"MoveFromFtpToWebDav process run source = %s , target = %s" %
(source_path, target_path))
target_webdav = WebDavConnection.create(
self.login, self.target.get('server_id'), self.logger)
source_ftp = FTPConnection.create(self.login,
self.source.get('server_id'),
self.logger)
t_total = threading.Thread(target=self.get_total,
args=(self.operation_progress,
self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(
temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(
uploading_path, target_path, self.overwrite,
file_basename, self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_ftp.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
next_tick = time.time() + REQUEST_DELAY
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
for path in self.paths:
try:
webdav.remove(path)
success_paths.append(path)
if time.time() > next_tick:
progress = {
'percent':
round(
float(len(success_paths)) /
float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) /
float(len(self.paths)), 2) * 100)) +
'%'
}
self.on_running(self.status_id,
progress=progress,
pid=self.pid,
pname=self.name)
next_tick = time.time() + REQUEST_DELAY
except Exception as e:
self.logger.error("Error removing file %s , error %s" %
(str(path), str(e)))
error_paths.append(path)
result = {"success": success_paths, "errors": error_paths}
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
source_path = self.get_abs_path(source_path)
webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
self.logger.info("MoveToWebDav process run source = %s , target = %s" % (source_path, target_path))
t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
t_total.start()
for path in self.paths:
try:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = abs_path
if os.path.isdir(abs_path):
uploading_path += '/'
file_basename += '/'
result_upload = webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
self.uploading_progress)
if result_upload['success']:
success_paths.append(path)
if os.path.isfile(abs_path):
os.remove(abs_path)
elif os.path.islink(abs_path):
os.unlink(abs_path)
elif os.path.isdir(abs_path):
shutil.rmtree(abs_path)
else:
error_paths.append(abs_path)
break
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def __init__(self, paths, session, *args, **kwargs):
super(CreateCopy, self).__init__(*args, **kwargs)
self.paths = paths
self.session = session
self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
def run(self):
try:
self.preload()
abs_path = self.path
self.logger.debug("FM WebDav WriteFile worker run(), abs_path = %s" % abs_path)
self.logger.debug("content %s" % pprint.pformat(self.content))
self.logger.debug("encoding %s" % pprint.pformat(self.encoding))
try:
decoded = self.content.decode('utf-8')
self.logger.debug("DECODED %s" % pprint.pformat(decoded))
except Exception as e:
self.logger.error("Error %s , %s" % (str(e), traceback.format_exc()))
raise e
webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
try:
hash_str = self.random_hash()
temp_dir = os.path.abspath(TMP_DIR + '/' + self.login + '/' + hash_str + '/')
os.makedirs(temp_dir)
filedir = webdav.parent(self.path)
filename = self.path
if filedir != '/':
filename = filename.replace(filedir, "/", 1)
temp_filename = temp_dir + filename
except Exception as e:
result = WebDavConnection.get_error(e,
"Unable to make file backup before saving \"%s\"." % os.path.basename(
abs_path))
self.on_error(result)
return
try:
content = decoded.encode(self.encoding)
self.create_local_file(temp_filename, content)
folder = webdav.parent(self.path)
webdav.upload(temp_filename, folder, True)
file_info = webdav.generate_file_info(self.path)
shutil.rmtree(temp_dir, True)
file_result = {
"encoding": self.encoding,
"item": file_info
}
result = {
"data": file_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
try:
webdav.upload(os.path.join(temp_dir, os.path.basename(abs_path)), abs_path, True)
except Exception as e:
result = WebDavConnection.get_error(e,
"Unable to upload tmp file during write error \"%s\"."
% os.path.basename(abs_path))
self.on_error(result)
return
result = WebDavConnection.get_error(e, "Unable to write file \"%s\"." % os.path.basename(abs_path))
self.on_error(result)
return
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def __init__(self, paths, session, *args, **kwargs):
super(ReadImages, self).__init__(*args, **kwargs)
self.paths = paths
self.session = session
self.webdav = WebDavConnection.create(self.login, self.session.get("server_id"), self.logger)
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
abs_path = path
source_path = webdav.parent(path)
file_basename = webdav.path(abs_path)
if webdav.isdir(abs_path):
destination = os.path.join(target_path, file_basename)
if not os.path.exists(destination):
os.makedirs(destination)
else:
raise Exception("destination already exist")
for file in webdav.listdir(abs_path):
current = file.encode("ISO-8859-1").decode("UTF-8")
relative_root = os.path.relpath(current, source_path)
if webdav.isdir(file):
d = file.encode("ISO-8859-1").decode("UTF-8")
target_dir = os.path.join(target_path,
relative_root, d)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
else:
raise Exception(
"destination dir already exists")
if webdav.isfile(file):
f = file.encode("ISO-8859-1").decode("UTF-8")
source_file = os.path.join(current, f)
target_file_path = os.path.join(
target_path, relative_root)
target_file = os.path.join(target_path,
relative_root, f)
if not os.path.exists(target_file):
download_result = webdav.download(
source_file, target_file_path)
if not download_result['success'] or len(
download_result['file_list']
['failed']) > 0:
raise download_result[
'error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception(
"destination file already exists")
elif webdav.isfile(abs_path):
try:
target_file = os.path.join(target_path, file_basename)
if not os.path.exists(target_file):
download_result = webdav.download(
abs_path, target_path)
if not download_result['success'] or len(
download_result['file_list']
['failed']) > 0:
raise download_result[
'error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
except Exception as e:
self.logger.info("Cannot copy file %s , %s" %
(abs_path, str(e)))
raise e
success_paths.append(path)
except Exception as e:
self.logger.error("Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
abs_path = path
source_path = webdav.parent(path)
file_basename = webdav.path(abs_path)
if webdav.isdir(abs_path):
destination = os.path.join(target_path, file_basename)
if not os.path.exists(destination):
os.makedirs(destination)
else:
raise Exception("destination already exist")
for file in webdav.listdir(abs_path):
current = file.encode("ISO-8859-1").decode("UTF-8")
relative_root = os.path.relpath(current, source_path)
if webdav.isdir(file):
d = file.encode("ISO-8859-1").decode("UTF-8")
target_dir = os.path.join(target_path, relative_root, d)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
else:
raise Exception("destination dir already exists")
if webdav.isfile(file):
f = file.encode("ISO-8859-1").decode("UTF-8")
source_file = os.path.join(current, f)
target_file_path = os.path.join(target_path, relative_root)
target_file = os.path.join(target_path, relative_root, f)
if not os.path.exists(target_file):
download_result = webdav.download(source_file, target_file_path)
if not download_result['success'] or len(
download_result['file_list']['failed']) > 0:
raise download_result['error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
elif webdav.isfile(abs_path):
try:
target_file = os.path.join(target_path, file_basename)
if not os.path.exists(target_file):
download_result = webdav.download(abs_path, target_path)
if not download_result['success'] or len(download_result['file_list']['failed']) > 0:
raise download_result['error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
except Exception as e:
self.logger.info("Cannot copy file %s , %s" % (abs_path, str(e)))
raise e
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("CopyFromSftpToWebDav process run source = %s , target = %s" % (source_path, target_path))
target_webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("CopyFromWebDavToSFTP process run source = %s , target = %s" % (source_path, target_path))
source_webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
target_sftp = SFTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_success, upload_error = self.upload_files_recursive_to_sftp(
filename, temp_path, target_path, target_sftp, operation_progress)
if path in upload_success:
success_paths.append(path)
shutil.rmtree(temp_path, True)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info(
"CopyBetweenWebDav process run source = %s , target = %s" %
(source_path, target_path))
source_webdav = WebDavConnection.create(
self.login, self.source.get('server_id'), self.logger)
target_webdav = WebDavConnection.create(
self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total,
args=(self.operation_progress,
self.paths))
t_total.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(
path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_result = self.upload_file_to_webdav(
read_path, target_path, filename, target_webdav)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
abs_path = self.path
self.logger.debug(
"FM WebDav WriteFile worker run(), abs_path = %s" % abs_path)
self.logger.debug("content %s" % pprint.pformat(self.content))
self.logger.debug("encoding %s" % pprint.pformat(self.encoding))
try:
decoded = self.content.decode('utf-8')
self.logger.debug("DECODED %s" % pprint.pformat(decoded))
except Exception as e:
self.logger.error("Error %s , %s" %
(str(e), traceback.format_exc()))
raise e
webdav = WebDavConnection.create(self.login,
self.session.get('server_id'),
self.logger)
try:
hash_str = self.random_hash()
temp_dir = os.path.abspath(TMP_DIR + '/' + self.login + '/' +
hash_str + '/')
os.makedirs(temp_dir)
filedir = webdav.parent(self.path)
filename = self.path
if filedir != '/':
filename = filename.replace(filedir, "/", 1)
temp_filename = temp_dir + filename
except Exception as e:
result = WebDavConnection.get_error(
e, "Unable to make file backup before saving \"%s\"." %
os.path.basename(abs_path))
self.on_error(result)
return
try:
content = decoded.encode(self.encoding)
self.create_local_file(temp_filename, content)
folder = webdav.parent(self.path)
webdav.upload(temp_filename, folder, True)
file_info = webdav.generate_file_info(self.path)
shutil.rmtree(temp_dir, True)
file_result = {"encoding": self.encoding, "item": file_info}
result = {
"data": file_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
try:
webdav.upload(
os.path.join(temp_dir, os.path.basename(abs_path)),
abs_path, True)
except Exception as e:
result = WebDavConnection.get_error(
e,
"Unable to upload tmp file during write error \"%s\"."
% os.path.basename(abs_path))
self.on_error(result)
return
result = WebDavConnection.get_error(
e, "Unable to write file \"%s\"." %
os.path.basename(abs_path))
self.on_error(result)
return
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_directory = self.target.get('path')
if source_path is None:
raise Exception("Source path empty")
if target_directory is None:
raise Exception("Target path empty")
self.logger.info("MoveWebDav process run source = %s , target = %s" % (source_path, target_directory))
webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
# sleep for a while for better total counting
time.sleep(REQUEST_DELAY)
t_progress = threading.Thread(target=update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
replaced_path = path
if source_path != '/':
replaced_path = path.replace(webdav.parent(path), "/", 1)
if target_directory != '/':
target_path = target_directory + replaced_path
else:
target_path = replaced_path
if webdav.isdir(path):
path += '/'
copy_result = webdav.move_file(path, webdav.path(target_path), overwrite=True)
if not copy_result['success'] or len(copy_result['file_list']['failed']) > 0:
raise copy_result['error'] if copy_result['error'] is not None else Exception(
"Upload error")
operation_progress["processed"] += 1
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
source_path = self.get_abs_path(source_path)
webdav = WebDavConnection.create(self.login,
self.target.get('server_id'),
self.logger)
self.logger.debug(
"CopyToWebDav process run source = %s , target = %s" %
(source_path, target_path))
t_total = threading.Thread(target=self.get_total,
args=(self.operation_progress,
self.paths))
t_total.start()
for path in self.paths:
try:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = abs_path
if os.path.isdir(abs_path):
uploading_path += '/'
file_basename += '/'
result_upload = webdav.upload(uploading_path, target_path,
self.overwrite,
file_basename,
self.uploading_progress)
if result_upload['success']:
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)