def run(self): try: self.preload() source_abs_path = os.path.abspath(self.source_path) target_abs_path = os.path.abspath(self.target_path) self.logger.debug("FM FTP NewFile worker run(), source_abs_path = %s" % source_abs_path) self.logger.debug("FM FTP NewFile worker run(), target_abs_path = %s" % target_abs_path) ftp_connection = FTPConnection.create(self.login, self.session.get("server_id"), self.logger) try: source_info = ftp_connection.file_info(source_abs_path) ftp_connection.rename(source_abs_path, target_abs_path) ftp_connection.clear_cache() target_info = ftp_connection.file_info(target_abs_path) ftp_result = {"source": source_info, "target": target_info} result = {"data": ftp_result, "error": False, "message": None, "traceback": None} self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, "Unable to rename source element.") self.on_error(result) except Exception as e: result = {"error": True, "message": str(e), "traceback": traceback.format_exc()} self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM FTP NewFile worker run(), abs_path = %s" % abs_path) ftp_connection = self.get_ftp_connection(self.session) try: if ftp_connection.path.exists(abs_path): raise Exception("File with target name already exists") pid = ftp_connection.open(abs_path, 'w') if pid: pid.close() info = ftp_connection.lstat(abs_path) result = { "name": abs_path, "mode": ftp_connection.getmode(info), "mtime": info.st_mtime } else: raise Exception('Cannot write file resource on FTP server') result = { "data": result, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, str(e)) return self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM FTP NewFile worker run(), abs_path = %s" % abs_path) ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger) try: if ftp_connection.path.exists(abs_path): raise Exception("File with target name already exists") pid = ftp_connection.open(abs_path, 'w') if pid: pid.close() info = ftp_connection.lstat(abs_path) result = { "name": abs_path, "mode": ftp_connection.getmode(info), "mtime": info.st_mtime } else: raise Exception('Cannot write file resource on FTP server') result = { "data": result, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, str(e)) return self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() source_abs_path = os.path.abspath(self.source_path) target_abs_path = os.path.abspath(self.target_path) self.logger.debug("FM FTP NewFile worker run(), source_abs_path = %s" % source_abs_path) self.logger.debug("FM FTP NewFile worker run(), target_abs_path = %s" % target_abs_path) ftp_connection = self.get_ftp_connection(self.session) try: source_info = ftp_connection.file_info(source_abs_path) ftp_connection.rename(source_abs_path, target_abs_path) ftp_connection.clear_cache() target_info = ftp_connection.file_info(target_abs_path) ftp_result = { "source": source_info, "target": target_info } result = { "data": ftp_result, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, "Unable to rename source element.") self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM FTP MakeDir worker run(), abs_path = %s" % abs_path) ftp_connection = self.get_ftp_connection(self.session) try: ftp_connection.mkdir(abs_path) info = ftp_connection.lstat(abs_path) fileinfo = { "name": abs_path, "mode": ftp_connection.getmode(info), "mtime": info.st_mtime } result = { "data": fileinfo, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, "File path already exists") self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM FTP ReadFile worker run(), abs_path = %s" % abs_path) ftp_connection = self.get_ftp_connection(self.session) try: with ftp_connection.open(abs_path) as fd: content = fd.read() # part of file content for charset detection part_content = content[0:self.charset_detect_buffer] + content[-self.charset_detect_buffer:] chardet_result = chardet.detect(part_content) detected = chardet_result["encoding"] confidence = chardet_result["confidence"] self.logger.debug("Detected encoding = %s (%s), %s" % (detected, confidence, abs_path)) # костыль пока не соберем нормальную версию libmagick >= 5.10 # https://github.com/ahupp/python-magic/issues/47 # # так же можно собрать uchardet от Mozilla, пока изучаю ее (тоже свои косяки), # кстати ее порт на python chardet мы юзаем, а их сайт уже мертв :( re_utf8 = re.compile('.*charset\s*=\s*utf\-8.*', re.UNICODE | re.IGNORECASE | re.MULTILINE) html_ext = ['htm', 'html', 'phtml', 'php', 'inc', 'tpl', 'xml'] file_ext = os.path.splitext(abs_path)[1][1:].strip().lower() self.logger.debug("File ext = %s" % file_ext) if confidence > 0.75 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING: if detected == "ISO-8859-7": detected = "windows-1251" if detected == "ISO-8859-2": detected = "utf-8" if detected == "ascii": detected = "utf-8" if detected == "MacCyrillic": detected = "windows-1251" # если все же ошиблись - костыль на указанный в файле charset if detected != FM.DEFAULT_ENCODING and file_ext in html_ext: result_of_search = re_utf8.search(part_content) self.logger.debug(result_of_search) if result_of_search is not None: self.logger.debug("matched utf-8 charset") detected = FM.DEFAULT_ENCODING else: self.logger.debug("not matched utf-8 charset") elif confidence > 0.60 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING: if detected == "ISO-8859-2": detected = "windows-1251" if detected == "MacCyrillic": detected = "windows-1251" # если все же ошиблись - костыль на указанный в файле charset if detected != FM.DEFAULT_ENCODING and file_ext in html_ext: result_of_search = re_utf8.search(part_content) self.logger.debug(result_of_search) if result_of_search is not None: self.logger.debug("matched utf-8 charset") detected = FM.DEFAULT_ENCODING else: self.logger.debug("not matched utf-8 charset") elif detected == 'windows-1251' or detected == FM.DEFAULT_ENCODING: pass else: detected = FM.DEFAULT_ENCODING encoding = detected if (detected or "").lower() in FM.encodings else FM.DEFAULT_ENCODING self.logger.debug("Result encoding = %s, %s" % (encoding, abs_path)) answer = { "item": ftp_connection.file_info(abs_path), "content": content, "encoding": encoding } result = { "data": answer, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error(e, "Unable to open file \"%s\"." % os.path.basename(abs_path)) self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM WriteFile worker run(), abs_path = %s" % abs_path) self.logger.debug("content %s" % pprint.pformat(self.content)) self.logger.debug("encoding %s" % pprint.pformat(self.encoding)) try: decoded = self.content.decode('utf-8') self.logger.debug("DECODED %s" % pprint.pformat(decoded)) except Exception as e: self.logger.error("Error %s , %s" % (str(e), traceback.format_exc())) raise e ftp_connection = self.get_ftp_connection(self.session) try: temp_dir = os.path.abspath('/tmp/fm/' + self.random_hash()) os.makedirs(temp_dir) ftp_connection.download(abs_path, temp_dir) except Exception as e: ftp_connection.close() result = FTPConnection.get_error( e, "Unable to make file backup before saving \"%s\"." % os.path.basename(abs_path)) self.on_error(result) return try: if isinstance(self.content, bytes): self.content = self.content.decode(self.encoding) with ftp_connection.open(abs_path, 'w', self.encoding) as fd: fd.write(self.content) file_info = ftp_connection.file_info(abs_path) ftp_connection.close() shutil.rmtree(temp_dir, True) file_result = {"encoding": self.encoding, "item": file_info} result = { "data": file_result, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: try: ftp_connection.upload( os.path.join(temp_dir, os.path.basename(abs_path)), abs_path, True) except Exception as e: ftp_connection.close() result = FTPConnection.get_error( e, "Unable to upload tmp file during write error \"%s\"." % os.path.basename(abs_path)) self.on_error(result) return ftp_connection.close() result = FTPConnection.get_error( e, "Unable to write file \"%s\"." % os.path.basename(abs_path)) self.on_error(result) return except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM WriteFile worker run(), abs_path = %s" % abs_path) self.logger.debug("content %s" % pprint.pformat(self.content)) self.logger.debug("encoding %s" % pprint.pformat(self.encoding)) try: decoded = self.content.decode('utf-8') self.logger.debug("DECODED %s" % pprint.pformat(decoded)) except Exception as e: self.logger.error("Error %s , %s" % (str(e), traceback.format_exc())) raise e ftp_connection = FTPConnection.create(self.login, self.session.get('server_id'), self.logger) try: temp_dir = os.path.abspath('/tmp/fm/' + self.random_hash()) os.makedirs(temp_dir) ftp_connection.download(abs_path, temp_dir) except Exception as e: ftp_connection.close() result = FTPConnection.get_error(e, "Unable to make file backup before saving \"%s\"." % os.path.basename( abs_path)) self.on_error(result) return try: if isinstance(self.content, bytes): self.content = self.content.decode(self.encoding) with ftp_connection.open(abs_path, 'w', self.encoding) as fd: fd.write(self.content) file_info = ftp_connection.file_info(abs_path) ftp_connection.close() shutil.rmtree(temp_dir, True) file_result = { "encoding": self.encoding, "item": file_info } result = { "data": file_result, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: try: ftp_connection.upload(os.path.join(temp_dir, os.path.basename(abs_path)), abs_path, True) except Exception as e: ftp_connection.close() result = FTPConnection.get_error(e, "Unable to upload tmp file during write error \"%s\"." % os.path.basename(abs_path)) self.on_error(result) return ftp_connection.close() result = FTPConnection.get_error(e, "Unable to write file \"%s\"." % os.path.basename(abs_path)) self.on_error(result) return except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)
def run(self): try: self.preload() abs_path = os.path.abspath(self.path) self.logger.debug("FM FTP ReadFile worker run(), abs_path = %s" % abs_path) ftp_connection = self.get_ftp_connection(self.session) try: with ftp_connection.open(abs_path) as fd: content = fd.read() # part of file content for charset detection part_content = content[0:self.charset_detect_buffer] + content[ -self.charset_detect_buffer:] chardet_result = chardet.detect(part_content) detected = chardet_result["encoding"] confidence = chardet_result["confidence"] self.logger.debug("Detected encoding = %s (%s), %s" % (detected, confidence, abs_path)) # костыль пока не соберем нормальную версию libmagick >= 5.10 # https://github.com/ahupp/python-magic/issues/47 # # так же можно собрать uchardet от Mozilla, пока изучаю ее (тоже свои косяки), # кстати ее порт на python chardet мы юзаем, а их сайт уже мертв :( re_utf8 = re.compile('.*charset\s*=\s*utf\-8.*', re.UNICODE | re.IGNORECASE | re.MULTILINE) html_ext = ['htm', 'html', 'phtml', 'php', 'inc', 'tpl', 'xml'] file_ext = os.path.splitext(abs_path)[1][1:].strip().lower() self.logger.debug("File ext = %s" % file_ext) if confidence > 0.75 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING: if detected == "ISO-8859-7": detected = "windows-1251" if detected == "ISO-8859-2": detected = "utf-8" if detected == "ascii": detected = "utf-8" if detected == "MacCyrillic": detected = "windows-1251" # если все же ошиблись - костыль на указанный в файле charset if detected != FM.DEFAULT_ENCODING and file_ext in html_ext: result_of_search = re_utf8.search(part_content) self.logger.debug(result_of_search) if result_of_search is not None: self.logger.debug("matched utf-8 charset") detected = FM.DEFAULT_ENCODING else: self.logger.debug("not matched utf-8 charset") elif confidence > 0.60 and detected != 'windows-1251' and detected != FM.DEFAULT_ENCODING: if detected == "ISO-8859-2": detected = "windows-1251" if detected == "MacCyrillic": detected = "windows-1251" # если все же ошиблись - костыль на указанный в файле charset if detected != FM.DEFAULT_ENCODING and file_ext in html_ext: result_of_search = re_utf8.search(part_content) self.logger.debug(result_of_search) if result_of_search is not None: self.logger.debug("matched utf-8 charset") detected = FM.DEFAULT_ENCODING else: self.logger.debug("not matched utf-8 charset") elif detected == 'windows-1251' or detected == FM.DEFAULT_ENCODING: pass else: detected = FM.DEFAULT_ENCODING encoding = detected if ( detected or "").lower() in FM.encodings else FM.DEFAULT_ENCODING self.logger.debug("Result encoding = %s, %s" % (encoding, abs_path)) answer = { "item": ftp_connection.file_info(abs_path), "content": content, "encoding": encoding } result = { "data": answer, "error": False, "message": None, "traceback": None } self.on_success(result) except Exception as e: result = FTPConnection.get_error( e, "Unable to open file \"%s\"." % os.path.basename(abs_path)) self.on_error(result) except Exception as e: result = { "error": True, "message": str(e), "traceback": traceback.format_exc() } self.on_error(result)