def run(self):
try:
self.preload()
source_abs_path = os.path.abspath(self.source_path)
target_abs_path = os.path.abspath(self.target_path)
self.logger.debug("FM FTP NewFile worker run(), source_abs_path = %s" % source_abs_path)
self.logger.debug("FM FTP NewFile worker run(), target_abs_path = %s" % target_abs_path)
ftp_connection = FTPConnection.create(self.login, self.session.get("server_id"), self.logger)
try:
source_info = ftp_connection.file_info(source_abs_path)
ftp_connection.rename(source_abs_path, target_abs_path)
ftp_connection.clear_cache()
target_info = ftp_connection.file_info(target_abs_path)
ftp_result = {"source": source_info, "target": target_info}
result = {"data": ftp_result, "error": False, "message": None, "traceback": None}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, "Unable to rename source element.")
self.on_error(result)
except Exception as e:
result = {"error": True, "message": str(e), "traceback": traceback.format_exc()}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP ListFiles worker run(), abs_path = %s" % abs_path)
ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
listing = ftp_connection.list(path=abs_path)
result = {
"data": listing,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def get_total(self,
progress_object,
paths,
count_dirs=True,
count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" %
(count_dirs, count_files))
ftp = FTPConnection.create(self.login, self.source.get('server_id'),
self.logger)
for path in paths:
try:
abs_path = ftp.path.abspath(path)
for current, dirs, files in ftp.ftp.walk(
ftp.to_string(abs_path)):
if count_files:
progress_object["total"] += len(files)
if ftp.isfile(abs_path):
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" %
(str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP NewFile worker run(), abs_path = %s" % abs_path)
ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
try:
if ftp_connection.path.exists(abs_path):
raise Exception("File with target name already exists")
pid = ftp_connection.open(abs_path, 'w')
if pid:
pid.close()
info = ftp_connection.lstat(abs_path)
result = {
"name": abs_path,
"mode": ftp_connection.getmode(info),
"mtime": info.st_mtime
}
else:
raise Exception('Cannot write file resource on FTP server')
result = {
"data": result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, str(e))
return self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
next_tick = time.time() + REQUEST_DELAY
ftp = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
for path in self.paths:
try:
abs_path = ftp.path.abspath(path)
ftp.remove(abs_path)
success_paths.append(path)
if time.time() > next_tick:
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
self.on_running(self.status_id, progress=progress, pid=self.pid, pname=self.name)
next_tick = time.time() + REQUEST_DELAY
except Exception as e:
self.logger.error("Error removing file %s , error %s" % (str(path), str(e)))
error_paths.append(path)
result = {
"success": success_paths,
"errors": error_paths
}
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
self.logger.info("FTP UploadFile process run")
ftp = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
target_file = ftp.path.join(self.path, os.path.basename(self.file_path))
if not ftp.path.exists(target_file):
upload_result = ftp.upload(self.file_path, self.path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
os.remove(self.file_path)
elif self.overwrite and ftp.path.exists(target_file) and not ftp.path.isdir(target_file):
upload_result = ftp.upload(self.file_path, self.path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
os.remove(self.file_path)
elif self.overwrite and ftp.path.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
ftp.remove(target_file)
upload_result = ftp.upload(self.file_path, self.path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
os.remove(self.file_path)
else:
pass
result = {
"success": True
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP NewFile worker run(), abs_path = %s" %
abs_path)
ftp_connection = self.get_ftp_connection(self.session)
try:
if ftp_connection.path.exists(abs_path):
raise Exception("File with target name already exists")
pid = ftp_connection.open(abs_path, 'w')
if pid:
pid.close()
info = ftp_connection.lstat(abs_path)
result = {
"name": abs_path,
"mode": ftp_connection.getmode(info),
"mtime": info.st_mtime
}
else:
raise Exception('Cannot write file resource on FTP server')
result = {
"data": result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, str(e))
return self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
source_abs_path = os.path.abspath(self.source_path)
target_abs_path = os.path.abspath(self.target_path)
self.logger.debug("FM FTP NewFile worker run(), source_abs_path = %s" % source_abs_path)
self.logger.debug("FM FTP NewFile worker run(), target_abs_path = %s" % target_abs_path)
ftp_connection = self.get_ftp_connection(self.session)
try:
source_info = ftp_connection.file_info(source_abs_path)
ftp_connection.rename(source_abs_path, target_abs_path)
ftp_connection.clear_cache()
target_info = ftp_connection.file_info(target_abs_path)
ftp_result = {
"source": source_info,
"target": target_info
}
result = {
"data": ftp_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, "Unable to rename source element.")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" % abs_path)
ftp = FTPConnection.create(self.login, self.session.get("server_id"), self.logger)
htaccess_path = os.path.join(self.path, ".htaccess")
if not ftp.exists(htaccess_path):
default_rules = {
"allow_all": True,
"deny_all": False,
"order": "Allow,Deny",
"denied": [],
"allowed": [],
}
result = {"data": default_rules, "error": False, "message": None, "traceback": None}
self.on_success(result)
return
with ftp.open(abs_path) as fd:
content = fd.read()
htaccess = HtAccess(content, self.logger)
answer = htaccess.parse_file_content()
answer["allowed"] = htaccess.get_htaccess_allowed_ip()
answer["denied"] = htaccess.get_htaccess_denied_ip()
result = {"data": answer, "error": False, "message": None, "traceback": None}
self.on_success(result)
except Exception as e:
result = {"error": True, "message": str(e), "traceback": traceback.format_exc()}
self.on_error(result)
def get_total(self, progress_object, paths, count_dirs=True, count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" % (count_dirs, count_files))
ftp = FTPConnection.create(self.login, self.source.get('server_id'), self.logger)
for path in paths:
try:
abs_path = ftp.path.abspath(path)
for current, dirs, files in ftp.ftp.walk(ftp.to_string(abs_path)):
if count_files:
progress_object["total"] += len(files)
if ftp.isfile(abs_path):
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def run(self):
try:
self.preload()
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" % abs_path)
ftp = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
htaccess_path = os.path.join(self.path, '.htaccess')
if not ftp.exists(htaccess_path):
fd = ftp.open(htaccess_path, 'w')
fd.close()
with ftp.open(htaccess_path, 'r') as fd:
old_content = fd.read()
htaccess = HtAccess(old_content, self.logger)
content = htaccess.write_htaccess_file(self.params)
with ftp.open(htaccess_path, 'w') as fd:
fd.write(content)
result = {
"data": self.params,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP MakeDir worker run(), abs_path = %s" % abs_path)
ftp_connection = self.get_ftp_connection(self.session)
try:
ftp_connection.mkdir(abs_path)
info = ftp_connection.lstat(abs_path)
fileinfo = {
"name": abs_path,
"mode": ftp_connection.getmode(info),
"mtime": info.st_mtime
}
result = {
"data": fileinfo,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, "File path already exists")
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def make_listing(self):
if self.session_type == Module.HOME:
path = self.path if self.path is not None else self.get_home_dir()
abs_path = self.get_abs_path(path)
items = []
self.__list_recursive(abs_path, items, 1)
result = {
'path': self.get_rel_path(abs_path),
'items': items
}
return result
if self.session_type == Module.PUBLIC_FTP:
self.logger.info("FTP Listing preload")
path = self.path if self.path is not None else '/'
abs_path = os.path.abspath(path)
ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
listing = ftp_connection.list(path=abs_path)
return listing
raise Exception("Unknown session type")
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP ReadFile worker run(), abs_path = %s" %
abs_path)
ftp_connection = self.get_ftp_connection(self.session)
try:
with ftp_connection.open(abs_path) as fd:
content = fd.read()
# part of file content for charset detection
part_content = content[0:self.charset_detect_buffer] + content[
-self.charset_detect_buffer:]
chardet_result = chardet.detect(part_content)
detected = chardet_result["encoding"]
confidence = chardet_result["confidence"]
self.logger.debug("Detected encoding = %s (%s), %s" %
(detected, confidence, abs_path))
# костыль пока не соберем нормальную версию libmagick >= 5.10
# https://github.com/ahupp/python-magic/issues/47
#
# так же можно собрать uchardet от Mozilla, пока изучаю ее (тоже свои косяки),
# кстати ее порт на python chardet мы юзаем, а их сайт уже мертв :(
re_utf8 = re.compile('.*charset\s*=\s*utf\-8.*',
re.UNICODE | re.IGNORECASE | re.MULTILINE)
html_ext = ['htm', 'html', 'phtml', 'php', 'inc', 'tpl', 'xml']
file_ext = os.path.splitext(abs_path)[1][1:].strip().lower()
self.logger.debug("File ext = %s" % file_ext)
if confidence > 0.75 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING:
if detected == "ISO-8859-7":
detected = "windows-1251"
if detected == "ISO-8859-2":
detected = "utf-8"
if detected == "ascii":
detected = "utf-8"
if detected == "MacCyrillic":
detected = "windows-1251"
# если все же ошиблись - костыль на указанный в файле charset
if detected != FM.DEFAULT_ENCODING and file_ext in html_ext:
result_of_search = re_utf8.search(part_content)
self.logger.debug(result_of_search)
if result_of_search is not None:
self.logger.debug("matched utf-8 charset")
detected = FM.DEFAULT_ENCODING
else:
self.logger.debug("not matched utf-8 charset")
elif confidence > 0.60 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING:
if detected == "ISO-8859-2":
detected = "windows-1251"
if detected == "MacCyrillic":
detected = "windows-1251"
# если все же ошиблись - костыль на указанный в файле charset
if detected != FM.DEFAULT_ENCODING and file_ext in html_ext:
result_of_search = re_utf8.search(part_content)
self.logger.debug(result_of_search)
if result_of_search is not None:
self.logger.debug("matched utf-8 charset")
detected = FM.DEFAULT_ENCODING
else:
self.logger.debug("not matched utf-8 charset")
elif detected == 'windows-1251' or detected == FM.DEFAULT_ENCODING:
pass
else:
detected = FM.DEFAULT_ENCODING
encoding = detected if (
detected
or "").lower() in FM.encodings else FM.DEFAULT_ENCODING
self.logger.debug("Result encoding = %s, %s" %
(encoding, abs_path))
answer = {
"item": ftp_connection.file_info(abs_path),
"content": content,
"encoding": encoding
}
result = {
"data": answer,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(
e,
"Unable to open file \"%s\"." % os.path.basename(abs_path))
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def get_ftp_connection(self, session):
return FTPConnection.create(self.login, session['server_id'], self.logger)
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
ftp = FTPConnection.create(self.login, self.source.get('server_id'), self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
abs_path = ftp.path.abspath(path)
source_path = ftp.path.dirname(path)
file_basename = ftp.path.basename(abs_path)
if ftp.isdir(abs_path):
destination = os.path.join(target_path, file_basename)
if not os.path.exists(destination):
os.makedirs(destination)
else:
raise Exception("destination already exist")
for current, dirs, files in ftp.ftp.walk(ftp.to_string(abs_path)):
current = current.encode("ISO-8859-1").decode("UTF-8")
relative_root = os.path.relpath(current, source_path)
for d in dirs:
d = d.encode("ISO-8859-1").decode("UTF-8")
target_dir = os.path.join(target_path, relative_root, d)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
else:
raise Exception("destination dir already exists")
for f in files:
f = f.encode("ISO-8859-1").decode("UTF-8")
source_file = os.path.join(current, f)
target_file_path = os.path.join(target_path, relative_root)
target_file = os.path.join(target_path, relative_root, f)
if not os.path.exists(target_file):
download_result = ftp.download(source_file, target_file_path)
if not download_result['success'] or len(
download_result['file_list']['failed']) > 0:
raise download_result['error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
elif ftp.isfile(abs_path):
try:
target_file = os.path.join(target_path, file_basename)
if not os.path.exists(target_file):
download_result = ftp.download(abs_path, target_path)
if not download_result['success'] or len(download_result['file_list']['failed']) > 0:
raise download_result['error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
except Exception as e:
self.logger.info("Cannot copy file %s , %s" % (abs_path, str(e)))
raise e
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def get_ftp_connection(self, session):
return FTPConnection.create(self.login, session['server_id'],
self.logger)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("CopyFromFtp process run source = %s , target = %s" % (source_path, target_path))
source_ftp = FTPConnection.create(self.login, self.source.get('server_id'), self.logger)
target_ftp = FTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
abs_path = source_ftp.path.abspath(path)
file_basename = source_ftp.path.basename(abs_path)
if source_ftp.isdir(abs_path):
destination = target_ftp.path.join(target_path, file_basename)
if not target_ftp.exists(destination):
target_ftp.makedirs(destination)
elif self.overwrite and target_ftp.exists(destination) and not target_ftp.isdir(destination):
target_ftp.remove(destination)
target_ftp.makedirs(destination)
elif not self.overwrite and target_ftp.exists(destination) and not target_ftp.isdir(
destination):
raise Exception("destination is not a dir")
else:
pass
operation_progress["processed"] += 1
for current, dirs, files in source_ftp.ftp.walk(source_ftp.to_string(abs_path)):
current = current.encode("ISO-8859-1").decode("UTF-8")
relative_root = target_ftp.relative_root(source_path, current)
for d in dirs:
d = d.encode("ISO-8859-1").decode("UTF-8")
target_dir = target_ftp.path.join(target_path, relative_root, d)
if not target_ftp.exists(target_dir):
target_ftp.makedirs(target_dir)
elif self.overwrite and target_ftp.exists(target_dir) and not target_ftp.isdir(
target_dir):
target_ftp.remove(target_dir)
target_ftp.makedirs(target_dir)
elif not self.overwrite and target_ftp.exists(target_dir) and not target_ftp.isdir(
target_dir):
raise Exception("destination is not a dir")
else:
pass
operation_progress["processed"] += 1
for f in files:
f = f.encode("ISO-8859-1").decode("UTF-8")
source_file = source_ftp.path.join(current, f)
target_file = target_ftp.path.join(target_path, relative_root, f)
if not target_ftp.exists(target_file):
transfer_between_ftp(source_ftp, target_ftp, source_file, target_file)
elif self.overwrite and target_ftp.exists(target_file) and not target_ftp.isdir(
target_file):
transfer_between_ftp(source_ftp, target_ftp, source_file, target_file)
elif self.overwrite and target_ftp.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
target_ftp.remove(target_file)
transfer_between_ftp(source_ftp, target_ftp, source_file, target_file)
else:
pass
operation_progress["processed"] += 1
elif source_ftp.isfile(abs_path):
try:
target_file = target_ftp.path.join(target_path, file_basename)
if not target_ftp.exists(target_file):
transfer_between_ftp(source_ftp, target_ftp, abs_path, target_file)
elif self.overwrite and target_ftp.exists(target_file) and not target_ftp.isdir(
target_file):
transfer_between_ftp(source_ftp, target_ftp, abs_path, target_file)
elif self.overwrite and target_ftp.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
target_ftp.remove(target_file)
transfer_between_ftp(source_ftp, target_ftp, abs_path, target_file)
else:
pass
except Exception as e:
self.logger.info("Cannot copy file %s , %s" % (abs_path, str(e)))
raise e
finally:
operation_progress["processed"] += 1
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
self.logger.info("CreateCopy process run")
ftp = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
# Временная хеш таблица для директорий по которым будем делать листинг
directories = {}
for path in self.paths:
dirname = ftp.path.dirname(path)
if dirname not in directories.keys():
directories[dirname] = []
directories[dirname].append(path)
# Массив хешей source -> target для каждого пути
copy_paths = []
# Эта содомия нужна чтобы составтить массив source -> target для создания копии файла с красивым именем
# с учетом того что могут быть совпадения
for dirname, dir_paths in directories.items():
dir_listing = ftp.listdir(dirname)
for dir_path in dir_paths:
i = 0
exist = False
if ftp.isdir(dir_path):
filename = ftp.path.basename(dir_path)
ext = ''
else:
filename, file_extension = ftp.path.splitext(dir_path)
ext = file_extension
copy_name = filename + ' copy' + ext if i == 0 else filename + ' copy(' + str(i) + ')' + ext
for dir_current_path in dir_listing:
if copy_name == dir_current_path:
exist = True
i += 1
break
if not exist:
copy_paths.append({
'source': dir_path,
'target': ftp.path.join(dirname, copy_name)
})
while exist:
exist = False
if ftp.isdir(dir_path):
filename = ftp.path.basename(dir_path)
ext = ''
else:
filename, file_extension = ftp.path.splitext(dir_path)
ext = file_extension
copy_name = filename + ' copy' + ext if i == 0 else filename + ' copy(' + str(i) + ')' + ext
for dir_current_path in dir_listing:
if copy_name == dir_current_path:
exist = True
i += 1
break
if not exist:
dir_listing.append(copy_name)
copy_paths.append({
'source': dir_path,
'target': os.path.join(dirname, copy_name)
})
success_paths = []
error_paths = []
created_paths = []
next_tick = time.time() + REQUEST_DELAY
for copy_path in copy_paths:
try:
source_path = copy_path.get('source')
target_path = copy_path.get('target')
if ftp.isfile(source_path):
copy_result = ftp.copy_file(source_path, ftp.path.dirname(target_path), overwrite=True,
rename=target_path)
if not copy_result['success'] or len(copy_result['file_list']['failed']) > 0:
raise copy_result['error'] if copy_result['error'] is not None else Exception(
"Upload error")
elif ftp.isdir(source_path):
copy_result = ftp.copy_dir(source_path, ftp.path.dirname(target_path), overwrite=True,
rename=target_path)
if not copy_result['success'] or len(copy_result['file_list']['failed']) > 0:
raise copy_result['error'] if copy_result['error'] is not None else Exception(
"Upload error")
else:
error_paths.append(source_path)
break
success_paths.append(source_path)
created_paths.append(ftp.file_info(target_path))
if time.time() > next_tick:
progress = {
'percent': round(float(len(success_paths)) / float(len(copy_paths)), 2),
'text': str(
int(round(float(len(success_paths)) / float(len(copy_paths)), 2) * 100)) + '%'
}
self.on_running(self.status_id, progress=progress, pid=self.pid, pname=self.name)
next_tick = time.time() + REQUEST_DELAY
except Exception as e:
self.logger.error("Error copy file %s , error %s" % (str(source_path), str(e)))
error_paths.append(source_path)
result = {
"success": success_paths,
"errors": error_paths,
"items": created_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(copy_paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(copy_paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
ftp = FTPConnection.create(self.login, self.source.get('server_id'),
self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
abs_path = ftp.path.abspath(path)
source_path = ftp.path.dirname(path)
file_basename = ftp.path.basename(abs_path)
if ftp.isdir(abs_path):
destination = os.path.join(target_path, file_basename)
if not os.path.exists(destination):
os.makedirs(destination)
else:
raise Exception("destination already exist")
for current, dirs, files in ftp.ftp.walk(
ftp.to_string(abs_path)):
current = current.encode("ISO-8859-1").decode("UTF-8")
relative_root = os.path.relpath(current, source_path)
for d in dirs:
d = d.encode("ISO-8859-1").decode("UTF-8")
target_dir = os.path.join(target_path,
relative_root, d)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
else:
raise Exception(
"destination dir already exists")
for f in files:
f = f.encode("ISO-8859-1").decode("UTF-8")
source_file = os.path.join(current, f)
target_file_path = os.path.join(
target_path, relative_root)
target_file = os.path.join(target_path,
relative_root, f)
if not os.path.exists(target_file):
download_result = ftp.download(
source_file, target_file_path)
if not download_result['success'] or len(
download_result['file_list']
['failed']) > 0:
raise download_result[
'error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception(
"destination file already exists")
elif ftp.isfile(abs_path):
try:
target_file = os.path.join(target_path, file_basename)
if not os.path.exists(target_file):
download_result = ftp.download(
abs_path, target_path)
if not download_result['success'] or len(
download_result['file_list']
['failed']) > 0:
raise download_result[
'error'] if download_result[
'error'] is not None else Exception(
"Download error")
else:
raise Exception("destination file already exists")
except Exception as e:
self.logger.info("Cannot copy file %s , %s" %
(abs_path, str(e)))
raise e
success_paths.append(path)
except Exception as e:
self.logger.error("Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info(
"CopyFromWebDavToFTP process run source = %s , target = %s" %
(source_path, target_path))
source_webdav = WebDavConnection.create(
self.login, self.source.get('server_id'), self.logger)
target_ftp = FTPConnection.create(self.login,
self.target.get('server_id'),
self.logger)
t_total = threading.Thread(target=self.get_total,
args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress,
args=(operation_progress, ))
t_progress.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(
path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_success, upload_error = self.upload_files_recursive_to_ftp(
filename, temp_path, target_path, target_ftp,
operation_progress)
if path in upload_success:
success_paths.append(path)
shutil.rmtree(temp_path, True)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
source_path = self.get_abs_path(source_path)
self.logger.info("CopyToFtp process run source = %s , target = %s" % (source_path, target_path))
ftp = FTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
if os.path.isdir(abs_path):
destination = ftp.path.join(target_path, file_basename)
if not ftp.exists(destination):
ftp.mkdir(destination)
elif self.overwrite and ftp.exists(destination) and not ftp.isdir(destination):
ftp.remove(destination)
ftp.mkdir(destination)
elif not self.overwrite and ftp.exists(destination) and not ftp.isdir(destination):
raise Exception("destination is not a dir")
else:
pass
operation_progress["processed"] += 1
for current, dirs, files in os.walk(abs_path):
relative_root = os.path.relpath(current, source_path)
for d in dirs:
target_dir = ftp.path.join(target_path, relative_root, d)
if not ftp.exists(target_dir):
ftp.mkdir(target_dir)
elif self.overwrite and ftp.exists(target_dir) and not ftp.isdir(target_dir):
ftp.remove(target_dir)
ftp.mkdir(target_dir)
elif not self.overwrite and os.path.exists(target_dir) and not ftp.isdir(
target_dir):
raise Exception("destination is not a dir")
else:
pass
operation_progress["processed"] += 1
for f in files:
source_file = os.path.join(current, f)
target_file_path = ftp.path.join(target_path, relative_root)
target_file = ftp.path.join(target_path, relative_root, f)
if not ftp.exists(target_file):
upload_result = ftp.upload(source_file, target_file_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception(
"Upload error")
elif self.overwrite and ftp.exists(target_file) and not ftp.isdir(target_file):
upload_result = ftp.upload(source_file, target_file_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception(
"Upload error")
elif self.overwrite and ftp.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
ftp.remove(target_file)
upload_result = ftp.upload(source_file, target_file_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result[
'error'] is not None else Exception(
"Upload error")
else:
pass
operation_progress["processed"] += 1
elif os.path.isfile(abs_path):
try:
target_file = ftp.path.join(target_path, file_basename)
if not ftp.exists(target_file):
upload_result = ftp.upload(abs_path, target_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
elif self.overwrite and ftp.exists(target_file) and not ftp.isdir(target_file):
upload_result = ftp.upload(abs_path, target_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
elif self.overwrite and ftp.isdir(target_file):
"""
See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
In case copy file when destination is dir
"""
ftp.remove(target_file)
upload_result = ftp.upload(abs_path, target_path)
if not upload_result['success'] or len(upload_result['file_list']['failed']) > 0:
raise upload_result['error'] if upload_result['error'] is not None else Exception(
"Upload error")
else:
pass
operation_progress["processed"] += 1
except Exception as e:
self.logger.info("Cannot copy file %s , %s" % (abs_path, str(e)))
raise e
finally:
operation_progress["processed"] += 1
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("MoveFromFtpToWebDav process run source = %s , target = %s" % (source_path, target_path))
target_webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
source_ftp = FTPConnection.create(self.login, self.source.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_ftp.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
paths_list = self.params.get('paths')
paths = []
for item in paths_list:
paths.append(item.get("path"))
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
recursive = self.params.get("recursive")
mode = int(self.params.get("code"), 8)
self.logger.info("ChmodFiles process run recursive = %s , mode = %s" % (recursive, mode))
recursive_dirs = False
recursive_files = False
ftp = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
if recursive:
recursive_dirs = False if self.params.get("recursive_mode") == 'files' else True
recursive_files = False if self.params.get("recursive_mode") == 'dirs' else True
t_total = threading.Thread(target=self.get_total,
args=(operation_progress, paths, recursive_dirs, recursive_files))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in paths:
try:
abs_path = ftp.path.abspath(path)
self.logger.debug("Changing attributes file %s , %s" % (str(abs_path), str(oct(mode))))
if recursive:
if recursive_dirs:
ftp.chmod(abs_path, mode)
operation_progress["processed"] += 1
for current, dirs, files in ftp.ftp.walk(abs_path):
if recursive_dirs:
for d in dirs:
dir_path = ftp.path.join(current, d)
if ftp.path.islink(dir_path):
try:
ftp.chmod(dir_path, mode)
except Exception as e:
self.logger.info("Cannot change attributes on symlink dir %s , %s, %s" % (
str(dir_path), str(oct(mode)), str(e)))
pass
else:
ftp.chmod(dir_path, mode)
operation_progress["processed"] += 1
if recursive_files:
for f in files:
file_path = ftp.path.join(current, f)
if ftp.path.islink(file_path):
try:
ftp.chmod(file_path, mode)
except Exception as e:
self.logger.info("Cannot change attributes on symlink file %s , %s, %s" % (
str(file_path), str(oct(mode)), str(e)))
pass
else:
ftp.chmod(file_path, mode)
operation_progress["processed"] += 1
else:
ftp.chmod(abs_path, mode)
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error change attributes file %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
progress = {
'percent': round(float(len(success_paths)) / float(len(paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(paths)), 2) * 100)) + '%'
}
ftp.ftp.close()
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM WriteFile worker run(), abs_path = %s" % abs_path)
self.logger.debug("content %s" % pprint.pformat(self.content))
self.logger.debug("encoding %s" % pprint.pformat(self.encoding))
try:
decoded = self.content.decode('utf-8')
self.logger.debug("DECODED %s" % pprint.pformat(decoded))
except Exception as e:
self.logger.error("Error %s , %s" % (str(e), traceback.format_exc()))
raise e
ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger)
try:
temp_dir = os.path.abspath('/tmp/fm/' + self.random_hash())
os.makedirs(temp_dir)
ftp_connection.download(abs_path, temp_dir)
except Exception as e:
ftp_connection.close()
result = FTPConnection.get_error(e,
"Unable to make file backup before saving \"%s\"." % os.path.basename(
abs_path))
self.on_error(result)
return
try:
if isinstance(self.content, bytes):
self.content = self.content.decode(self.encoding)
with ftp_connection.open(abs_path, 'w', self.encoding) as fd:
fd.write(self.content)
file_info = ftp_connection.file_info(abs_path)
ftp_connection.close()
shutil.rmtree(temp_dir, True)
file_result = {
"encoding": self.encoding,
"item": file_info
}
result = {
"data": file_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
try:
ftp_connection.upload(os.path.join(temp_dir, os.path.basename(abs_path)), abs_path, True)
except Exception as e:
ftp_connection.close()
result = FTPConnection.get_error(e,
"Unable to upload tmp file during write error \"%s\"."
% os.path.basename(abs_path))
self.on_error(result)
return
ftp_connection.close()
result = FTPConnection.get_error(e, "Unable to write file \"%s\"." % os.path.basename(abs_path))
self.on_error(result)
return
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("MoveFromWebDavToFTP process run source = %s , target = %s" % (source_path, target_path))
source_webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
target_ftp = FTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_success, upload_error = self.upload_files_recursive_to_ftp(
filename, temp_path, target_path, target_ftp, operation_progress)
if len(upload_error) == 0:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_webdav.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info(
"MoveFromFtpToWebDav process run source = %s , target = %s" %
(source_path, target_path))
target_webdav = WebDavConnection.create(
self.login, self.target.get('server_id'), self.logger)
source_ftp = FTPConnection.create(self.login,
self.source.get('server_id'),
self.logger)
t_total = threading.Thread(target=self.get_total,
args=(self.operation_progress,
self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(
temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(
uploading_path, target_path, self.overwrite,
file_basename, self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_ftp.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM FTP ReadFile worker run(), abs_path = %s" % abs_path)
ftp_connection = self.get_ftp_connection(self.session)
try:
with ftp_connection.open(abs_path) as fd:
content = fd.read()
# part of file content for charset detection
part_content = content[0:self.charset_detect_buffer] + content[-self.charset_detect_buffer:]
chardet_result = chardet.detect(part_content)
detected = chardet_result["encoding"]
confidence = chardet_result["confidence"]
self.logger.debug("Detected encoding = %s (%s), %s" % (detected, confidence, abs_path))
# костыль пока не соберем нормальную версию libmagick >= 5.10
# https://github.com/ahupp/python-magic/issues/47
#
# так же можно собрать uchardet от Mozilla, пока изучаю ее (тоже свои косяки),
# кстати ее порт на python chardet мы юзаем, а их сайт уже мертв :(
re_utf8 = re.compile('.*charset\s*=\s*utf\-8.*', re.UNICODE | re.IGNORECASE | re.MULTILINE)
html_ext = ['htm', 'html', 'phtml', 'php', 'inc', 'tpl', 'xml']
file_ext = os.path.splitext(abs_path)[1][1:].strip().lower()
self.logger.debug("File ext = %s" % file_ext)
if confidence > 0.75 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING:
if detected == "ISO-8859-7":
detected = "windows-1251"
if detected == "ISO-8859-2":
detected = "utf-8"
if detected == "ascii":
detected = "utf-8"
if detected == "MacCyrillic":
detected = "windows-1251"
# если все же ошиблись - костыль на указанный в файле charset
if detected != FM.DEFAULT_ENCODING and file_ext in html_ext:
result_of_search = re_utf8.search(part_content)
self.logger.debug(result_of_search)
if result_of_search is not None:
self.logger.debug("matched utf-8 charset")
detected = FM.DEFAULT_ENCODING
else:
self.logger.debug("not matched utf-8 charset")
elif confidence > 0.60 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING:
if detected == "ISO-8859-2":
detected = "windows-1251"
if detected == "MacCyrillic":
detected = "windows-1251"
# если все же ошиблись - костыль на указанный в файле charset
if detected != FM.DEFAULT_ENCODING and file_ext in html_ext:
result_of_search = re_utf8.search(part_content)
self.logger.debug(result_of_search)
if result_of_search is not None:
self.logger.debug("matched utf-8 charset")
detected = FM.DEFAULT_ENCODING
else:
self.logger.debug("not matched utf-8 charset")
elif detected == 'windows-1251' or detected == FM.DEFAULT_ENCODING:
pass
else:
detected = FM.DEFAULT_ENCODING
encoding = detected if (detected or "").lower() in FM.encodings else FM.DEFAULT_ENCODING
self.logger.debug("Result encoding = %s, %s" % (encoding, abs_path))
answer = {
"item": ftp_connection.file_info(abs_path),
"content": content,
"encoding": encoding
}
result = {
"data": answer,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
result = FTPConnection.get_error(e, "Unable to open file \"%s\"." % os.path.basename(abs_path))
self.on_error(result)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
try:
self.preload()
abs_path = os.path.abspath(self.path)
self.logger.debug("FM WriteFile worker run(), abs_path = %s" %
abs_path)
self.logger.debug("content %s" % pprint.pformat(self.content))
self.logger.debug("encoding %s" % pprint.pformat(self.encoding))
try:
decoded = self.content.decode('utf-8')
self.logger.debug("DECODED %s" % pprint.pformat(decoded))
except Exception as e:
self.logger.error("Error %s , %s" %
(str(e), traceback.format_exc()))
raise e
ftp_connection = self.get_ftp_connection(self.session)
try:
temp_dir = os.path.abspath('/tmp/fm/' + self.random_hash())
os.makedirs(temp_dir)
ftp_connection.download(abs_path, temp_dir)
except Exception as e:
ftp_connection.close()
result = FTPConnection.get_error(
e, "Unable to make file backup before saving \"%s\"." %
os.path.basename(abs_path))
self.on_error(result)
return
try:
if isinstance(self.content, bytes):
self.content = self.content.decode(self.encoding)
with ftp_connection.open(abs_path, 'w', self.encoding) as fd:
fd.write(self.content)
file_info = ftp_connection.file_info(abs_path)
ftp_connection.close()
shutil.rmtree(temp_dir, True)
file_result = {"encoding": self.encoding, "item": file_info}
result = {
"data": file_result,
"error": False,
"message": None,
"traceback": None
}
self.on_success(result)
except Exception as e:
try:
ftp_connection.upload(
os.path.join(temp_dir, os.path.basename(abs_path)),
abs_path, True)
except Exception as e:
ftp_connection.close()
result = FTPConnection.get_error(
e,
"Unable to upload tmp file during write error \"%s\"."
% os.path.basename(abs_path))
self.on_error(result)
return
ftp_connection.close()
result = FTPConnection.get_error(
e, "Unable to write file \"%s\"." %
os.path.basename(abs_path))
self.on_error(result)
return
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
