Beispiel #1
0
    def run(self):
        try:
            self.preload()

            source_abs_path = os.path.abspath(self.source_path)
            target_abs_path = os.path.abspath(self.target_path)

            self.logger.debug(
                "FM WebDav NewFile worker run(), source_abs_path = %s, target_abs_path = %s"
                % (source_abs_path, target_abs_path))

            webdav_connection = WebDavConnection.create(
                self.login, self.session.get('server_id'), self.logger)

            try:
                if webdav_connection.isdir(source_abs_path):
                    source_abs_path += '/'
                    target_abs_path += '/'
                source_info = webdav_connection.generate_file_info(
                    source_abs_path)
                webdav_connection.move_file(source_abs_path, target_abs_path)
                target_info = webdav_connection.generate_file_info(
                    target_abs_path)

                webdav_result = {"source": source_info, "target": target_info}

                result = {
                    "data": webdav_result,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)

            except Exception as e:
                result = WebDavConnection.get_error(
                    e, "Unable to rename source element.")
                self.on_error(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #2
0
    def get_total(self,
                  progress_object,
                  paths,
                  count_dirs=True,
                  count_files=True):
        self.logger.debug("start get_total() dirs = %s , files = %s" %
                          (count_dirs, count_files))
        webdav = WebDavConnection.create(self.login,
                                         self.target.get('server_id'),
                                         self.logger)
        for path in paths:
            try:
                if count_dirs:
                    progress_object["total"] += 1

                for file in webdav.listdir(path):
                    if webdav.isdir(file):
                        progress_object["total"] += 1
                    else:
                        progress_object["total"] += 1
            except Exception as e:
                self.logger.error("Error get_total file %s , error %s" %
                                  (str(path), str(e)))
                continue

        progress_object["total_done"] = True
        self.logger.debug("done get_total()")
        return
Beispiel #3
0
    def run(self):
        try:
            self.preload()
            abs_path = os.path.abspath(self.path)
            self.logger.debug(
                "FM WebDav ListFiles worker run(), abs_path = %s" % abs_path)

            webdav_connection = WebDavConnection.create(
                self.login, self.session.get('server_id'), self.logger)
            listing = webdav_connection.list(path=abs_path)

            result = {
                "data": listing,
                "error": False,
                "message": None,
                "traceback": None
            }

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #4
0
    def run(self):
        try:
            self.preload()
            abs_path = os.path.abspath(self.path)
            self.logger.debug("FM WebDav ListFiles worker run(), abs_path = %s" % abs_path)

            webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
            listing = webdav_connection.list(path=abs_path)

            result = {
                "data": listing,
                "error": False,
                "message": None,
                "traceback": None
            }

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #5
0
    def run(self):
        try:
            self.preload()
            abs_path = self.get_abs_path(self.path)
            self.logger.debug(
                "FM ReadRulesWebDav worker run(), abs_path = %s" % abs_path)

            webdav = WebDavConnection.create(self.login,
                                             self.session.get('server_id'),
                                             self.logger)

            htaccess_path = os.path.join(self.path, '.htaccess')

            if not webdav.exists(htaccess_path):
                default_rules = {
                    'allow_all': True,
                    'deny_all': False,
                    'order': 'Allow,Deny',
                    'denied': [],
                    'allowed': []
                }

                result = {
                    "data": default_rules,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)
                return

            with webdav.open(abs_path) as fd:
                content = fd.read()

            htaccess = HtAccess(content, self.logger)

            answer = htaccess.parse_file_content()

            answer['allowed'] = htaccess.get_htaccess_allowed_ip()
            answer['denied'] = htaccess.get_htaccess_denied_ip()

            result = {
                "data": answer,
                "error": False,
                "message": None,
                "traceback": None
            }

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #6
0
    def __init__(self, paths, session, *args, **kwargs):
        super(ReadImages, self).__init__(*args, **kwargs)

        self.paths = paths
        self.session = session
        self.webdav = WebDavConnection.create(self.login,
                                              self.session.get('server_id'),
                                              self.logger)
Beispiel #7
0
    def run(self):
        try:
            self.preload()

            source_abs_path = os.path.abspath(self.source_path)
            target_abs_path = os.path.abspath(self.target_path)

            self.logger.debug("FM WebDav NewFile worker run(), source_abs_path = %s, target_abs_path = %s" %
                              (source_abs_path, target_abs_path))

            webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

            try:
                if webdav_connection.isdir(source_abs_path):
                    source_abs_path += '/'
                    target_abs_path += '/'
                source_info = webdav_connection.generate_file_info(source_abs_path)
                webdav_connection.move_file(source_abs_path, target_abs_path)
                target_info = webdav_connection.generate_file_info(target_abs_path)

                webdav_result = {
                    "source": source_info,
                    "target": target_info
                }

                result = {
                    "data": webdav_result,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)

            except Exception as e:
                result = WebDavConnection.get_error(e, "Unable to rename source element.")
                self.on_error(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #8
0
    def __init__(self, path, session, *args, **kwargs):
        super(ReadFile, self).__init__(*args, **kwargs)

        self.path = path

        # чем больше - тем точнее определяется кодировка, но медленнее, 50000 - выбрано опытным путем
        self.charset_detect_buffer = 50000
        self.session = session
        self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
Beispiel #9
0
    def __init__(self, path, encoding, session, *args, **kwargs):
        super(ReadFile, self).__init__(*args, **kwargs)

        self.path = path
        self.encoding = encoding

        # чем больше - тем точнее определяется кодировка, но медленнее, 50000 - выбрано опытным путем
        self.charset_detect_buffer = 50000
        self.session = session
        self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
Beispiel #10
0
    def run(self):
        try:
            self.preload()
            self.logger.info("WebDav UploadFile process run")

            webdav = WebDavConnection.create(self.login,
                                             self.session.get('server_id'),
                                             self.logger)

            target_file = "{0}{1}".format(self.path,
                                          os.path.basename(self.file_path))

            if not webdav.exists(target_file):
                upload_result = webdav.upload(self.file_path, self.path)
                if not upload_result['success'] or len(
                        upload_result['file_list']['failed']) > 0:
                    raise upload_result['error'] if upload_result[
                        'error'] is not None else Exception("Upload error")
                os.remove(self.file_path)
            elif self.overwrite and webdav.exists(
                    target_file) and not webdav.isdir(target_file):
                upload_result = webdav.upload(self.file_path, self.path)
                if not upload_result['success'] or len(
                        upload_result['file_list']['failed']) > 0:
                    raise upload_result['error'] if upload_result[
                        'error'] is not None else Exception("Upload error")
                os.remove(self.file_path)
            elif self.overwrite and webdav.isdir(target_file):
                """
                See https://docs.python.org/3.4/library/shutil.html?highlight=shutil#shutil.copy
                In case copy file when destination is dir
                """
                webdav.remove(target_file)
                upload_result = webdav.upload(self.file_path, self.path)
                if not upload_result['success'] or len(
                        upload_result['file_list']['failed']) > 0:
                    raise upload_result['error'] if upload_result[
                        'error'] is not None else Exception("Upload error")
                os.remove(self.file_path)
            else:
                pass

            result = {"success": True}

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #11
0
    def run(self):
        try:
            self.preload()
            abs_path = os.path.abspath(self.path)
            self.logger.debug("FM WebDav MakeDir worker run(), abs_path = %s" %
                              abs_path)

            webdav_connection = WebDavConnection.create(
                self.login, self.session.get('server_id'), self.logger)

            try:
                webdav_connection.mkdir(abs_path)
                info = webdav_connection.info(abs_path)
                fileinfo = {
                    "name": abs_path,
                    "mode": webdav_connection.getmode(info),
                    "mtime": str(info['modified'])
                }

                result = {
                    "data": fileinfo,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)
            except Exception as e:
                result = WebDavConnection.get_error(
                    e, "File path already exists")
                self.on_error(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #12
0
    def get_total(self, progress_object, paths, count_files=True):
        self.logger.debug("start get_total() files = %s" % count_files)
        webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
        for path in paths:
            try:
                self.recursive_total(webdav, path, progress_object)

            except Exception as e:
                self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
                continue

        progress_object["total_done"] = True
        self.logger.debug("done get_total(), found %s objects" % progress_object.get("total"))
        return
    def get_total(self, progress_object, paths, count_files=True):
        self.logger.debug("start get_total() files = %s" % count_files)
        webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
        for path in paths:
            try:
                self.recursive_total(webdav, path, progress_object)

            except Exception as e:
                self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
                continue

        progress_object["total_done"] = True
        self.logger.debug("done get_total(), found %s objects" % progress_object.get("total"))
        return
Beispiel #14
0
    def run(self):
        try:
            self.preload()
            abs_path = os.path.abspath(self.path)
            self.logger.debug("FM WebDav MakeDir worker run(), abs_path = %s" % abs_path)

            webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

            try:
                webdav_connection.mkdir(abs_path)
                info = webdav_connection.info(abs_path)
                fileinfo = {
                    "name": abs_path,
                    "mode": webdav_connection.getmode(info),
                    "mtime": str(info['modified'])
                }

                result = {
                    "data": fileinfo,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)
            except Exception as e:
                result = WebDavConnection.get_error(e, "File path already exists")
                self.on_error(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #15
0
    def __init__(self, source, target, paths, overwrite, *args, **kwargs):
        super(MoveFromWebDav, self).__init__(*args, **kwargs)

        self.source = source
        self.target = target
        self.paths = paths
        self.overwrite = overwrite
        self.webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
        self.operation_progress = {
            "total_done": False,
            "total": 0,
            "operation_done": False,
            "processed": 0,
            "previous_percent": 0
        }
Beispiel #16
0
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            next_tick = time.time() + REQUEST_DELAY
            webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

            for path in self.paths:
                try:
                    webdav.remove(path)

                    success_paths.append(path)

                    if time.time() > next_tick:
                        progress = {
                            'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                            'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
                        }

                        self.on_running(self.status_id, progress=progress, pid=self.pid, pname=self.name)
                        next_tick = time.time() + REQUEST_DELAY

                except Exception as e:
                    self.logger.error("Error removing file %s , error %s" % (str(path), str(e)))
                    error_paths.append(path)

            result = {
                "success": success_paths,
                "errors": error_paths
            }

            progress = {
                'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
            }

            self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
Beispiel #17
0
    def __init__(self, source, target, paths, overwrite, *args, **kwargs):
        super(MoveFromWebDav, self).__init__(*args, **kwargs)

        self.source = source
        self.target = target
        self.paths = paths
        self.overwrite = overwrite
        self.webdav = WebDavConnection.create(self.login,
                                              self.source.get('server_id'),
                                              self.logger)
        self.operation_progress = {
            "total_done": False,
            "total": 0,
            "operation_done": False,
            "processed": 0,
            "previous_percent": 0
        }
Beispiel #18
0
    def run(self):
        try:
            self.preload()
            abs_path = self.get_abs_path(self.path)
            self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" %
                              abs_path)

            webdav = WebDavConnection.create(self.login,
                                             self.session.get('server_id'),
                                             self.logger)

            htaccess_path = os.path.join(self.path, '.htaccess')

            if not webdav.exists(htaccess_path):
                fd = webdav.open(htaccess_path, 'w')
                fd.close()

            with webdav.open(htaccess_path, 'r') as fd:
                old_content = fd.read()

            htaccess = HtAccess(old_content, self.logger)
            content = htaccess.write_htaccess_file(self.params)

            with webdav.open(htaccess_path, 'w') as fd:
                fd.write(content)

            result = {
                "data": self.params,
                "error": False,
                "message": None,
                "traceback": None
            }

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #19
0
    def get_total(self, progress_object, paths, count_dirs=True, count_files=True):
        self.logger.debug("start get_total() dirs = %s , files = %s" % (count_dirs, count_files))
        webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
        for path in paths:
            try:
                if count_dirs:
                    progress_object["total"] += 1

                for file in webdav.listdir(path):
                    if webdav.isdir(file):
                        progress_object["total"] += 1
                    else:
                        progress_object["total"] += 1
            except Exception as e:
                self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
                continue

        progress_object["total_done"] = True
        self.logger.debug("done get_total()")
        return
Beispiel #20
0
    def make_listing(self):

        if self.session_type == Module.HOME:
            path = self.path if self.path is not None else self.get_home_dir()
            abs_path = self.get_abs_path(path)
            items = []
            self.__list_recursive(abs_path, items, 1)
            result = {
                'path': self.get_rel_path(abs_path),
                'items': items
            }

            return result

        if self.session_type == Module.FTP:
            self.logger.info("FTP Listing preload")
            path = self.path if self.path is not None else '/'
            abs_path = os.path.abspath(path)
            ftp_connection = self.get_ftp_connection(self.session)
            listing = ftp_connection.list(path=abs_path)

            return listing

        if self.session_type == Module.SFTP:
            self.logger.info("SFTP Listing preload")
            path = self.path if self.path is not None else '.'
            sftp_connection = self.get_sftp_connection(self.session)
            listing = sftp_connection.list(path=path)

            return listing

        if self.session_type == Module.WEBDAV:
            self.logger.info("WebDav Listing preload")
            path = self.path if self.path is not None else '/'
            abs_path = os.path.abspath(path)
            webdav_connection = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
            listing = webdav_connection.list(path=abs_path)

            return listing

        raise Exception("Unknown session type")
    def run(self):
        try:
            self.preload()
            abs_path = self.get_abs_path(self.path)
            self.logger.debug("FM ReadRulesLocal worker run(), abs_path = %s" % abs_path)

            webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

            htaccess_path = os.path.join(self.path, '.htaccess')

            if not webdav.exists(htaccess_path):
                fd = webdav.open(htaccess_path, 'w')
                fd.close()

            with webdav.open(htaccess_path, 'r') as fd:
                old_content = fd.read()

            htaccess = HtAccess(old_content, self.logger)
            content = htaccess.write_htaccess_file(self.params)

            with webdav.open(htaccess_path, 'w') as fd:
                fd.write(content)

            result = {
                    "data": self.params,
                    "error": False,
                    "message": None,
                    "traceback": None
            }

            self.on_success(result)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            source_path = self.source.get('path')
            target_path = self.target.get('path')
            hash_str = self.random_hash()
            temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            self.logger.info(
                "MoveFromFtpToWebDav process run source = %s , target = %s" %
                (source_path, target_path))

            target_webdav = WebDavConnection.create(
                self.login, self.target.get('server_id'), self.logger)
            source_ftp = FTPConnection.create(self.login,
                                              self.source.get('server_id'),
                                              self.logger)
            t_total = threading.Thread(target=self.get_total,
                                       args=(self.operation_progress,
                                             self.paths))
            t_total.start()

            for path in self.paths:
                try:
                    success_paths, error_paths = self.copy_files_to_tmp(
                        temp_path)

                    if len(error_paths) == 0:
                        abs_path = self.get_abs_path(path)
                        file_basename = os.path.basename(abs_path)
                        uploading_path = temp_path + file_basename
                        if os.path.isdir(uploading_path):
                            uploading_path += '/'
                            file_basename += '/'

                        upload_result = target_webdav.upload(
                            uploading_path, target_path, self.overwrite,
                            file_basename, self.uploading_progress)

                        if upload_result['success']:
                            success_paths.append(path)
                            shutil.rmtree(temp_path, True)
                            source_ftp.remove(path)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" %
                        (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            self.operation_progress["operation_done"] = True

            result = {"success": success_paths, "errors": error_paths}

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent':
                round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text':
                str(
                    int(
                        round(
                            float(len(success_paths)) / float(len(self.paths)),
                            2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id,
                            data=result,
                            progress=progress,
                            pid=self.pid,
                            pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id,
                          result,
                          pid=self.pid,
                          pname=self.name)
Beispiel #23
0
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            next_tick = time.time() + REQUEST_DELAY
            webdav = WebDavConnection.create(self.login,
                                             self.session.get('server_id'),
                                             self.logger)

            for path in self.paths:
                try:
                    webdav.remove(path)

                    success_paths.append(path)

                    if time.time() > next_tick:
                        progress = {
                            'percent':
                            round(
                                float(len(success_paths)) /
                                float(len(self.paths)), 2),
                            'text':
                            str(
                                int(
                                    round(
                                        float(len(success_paths)) /
                                        float(len(self.paths)), 2) * 100)) +
                            '%'
                        }

                        self.on_running(self.status_id,
                                        progress=progress,
                                        pid=self.pid,
                                        pname=self.name)
                        next_tick = time.time() + REQUEST_DELAY

                except Exception as e:
                    self.logger.error("Error removing file %s , error %s" %
                                      (str(path), str(e)))
                    error_paths.append(path)

            result = {"success": success_paths, "errors": error_paths}

            progress = {
                'percent':
                round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text':
                str(
                    int(
                        round(
                            float(len(success_paths)) / float(len(self.paths)),
                            2) * 100)) + '%'
            }

            self.on_success(self.status_id,
                            data=result,
                            progress=progress,
                            pid=self.pid,
                            pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id,
                          result,
                          pid=self.pid,
                          pname=self.name)
Beispiel #24
0
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            source_path = self.source.get('path')
            target_path = self.target.get('path')

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            source_path = self.get_abs_path(source_path)
            webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)

            self.logger.info("MoveToWebDav process run source = %s , target = %s" % (source_path, target_path))

            t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
            t_total.start()

            for path in self.paths:
                try:
                    abs_path = self.get_abs_path(path)
                    file_basename = os.path.basename(abs_path)

                    uploading_path = abs_path
                    if os.path.isdir(abs_path):
                        uploading_path += '/'
                        file_basename += '/'

                    result_upload = webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
                                                  self.uploading_progress)

                    if result_upload['success']:
                        success_paths.append(path)
                        if os.path.isfile(abs_path):
                            os.remove(abs_path)
                        elif os.path.islink(abs_path):
                            os.unlink(abs_path)
                        elif os.path.isdir(abs_path):
                            shutil.rmtree(abs_path)
                        else:
                            error_paths.append(abs_path)
                            break

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            self.operation_progress["operation_done"] = True

            result = {
                "success": success_paths,
                "errors": error_paths
            }

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
Beispiel #25
0
    def __init__(self, paths, session, *args, **kwargs):
        super(CreateCopy, self).__init__(*args, **kwargs)

        self.paths = paths
        self.session = session
        self.webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)
Beispiel #26
0
    def run(self):
        try:
            self.preload()
            abs_path = self.path
            self.logger.debug("FM WebDav WriteFile worker run(), abs_path = %s" % abs_path)

            self.logger.debug("content %s" % pprint.pformat(self.content))
            self.logger.debug("encoding %s" % pprint.pformat(self.encoding))

            try:
                decoded = self.content.decode('utf-8')
                self.logger.debug("DECODED %s" % pprint.pformat(decoded))
            except Exception as e:
                self.logger.error("Error %s , %s" % (str(e), traceback.format_exc()))
                raise e

            webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

            try:
                hash_str = self.random_hash()
                temp_dir = os.path.abspath(TMP_DIR + '/' + self.login + '/' + hash_str + '/')
                os.makedirs(temp_dir)
                filedir = webdav.parent(self.path)
                filename = self.path
                if filedir != '/':
                    filename = filename.replace(filedir, "/", 1)
                temp_filename = temp_dir + filename
            except Exception as e:
                result = WebDavConnection.get_error(e,
                                                 "Unable to make file backup before saving \"%s\"." % os.path.basename(
                                                     abs_path))
                self.on_error(result)
                return

            try:
                content = decoded.encode(self.encoding)
                self.create_local_file(temp_filename, content)

                folder = webdav.parent(self.path)
                webdav.upload(temp_filename, folder, True)

                file_info = webdav.generate_file_info(self.path)

                shutil.rmtree(temp_dir, True)

                file_result = {
                    "encoding": self.encoding,
                    "item": file_info
                }

                result = {
                    "data": file_result,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)

            except Exception as e:
                try:
                    webdav.upload(os.path.join(temp_dir, os.path.basename(abs_path)), abs_path, True)
                except Exception as e:
                    result = WebDavConnection.get_error(e,
                                                     "Unable to upload tmp file during write error \"%s\"."
                                                     % os.path.basename(abs_path))
                    self.on_error(result)
                    return

                result = WebDavConnection.get_error(e, "Unable to write file \"%s\"." % os.path.basename(abs_path))
                self.on_error(result)
                return

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #27
0
    def __init__(self, paths, session, *args, **kwargs):
        super(ReadImages, self).__init__(*args, **kwargs)

        self.paths = paths
        self.session = session
        self.webdav = WebDavConnection.create(self.login, self.session.get("server_id"), self.logger)
Beispiel #28
0
    def copy_files_to_tmp(self, target_path):
        if not os.path.exists(target_path):
            os.makedirs(target_path)

        webdav = WebDavConnection.create(self.login,
                                         self.session.get('server_id'),
                                         self.logger)

        success_paths = []
        error_paths = []

        for path in self.paths:
            try:
                abs_path = path
                source_path = webdav.parent(path)
                file_basename = webdav.path(abs_path)

                if webdav.isdir(abs_path):
                    destination = os.path.join(target_path, file_basename)

                    if not os.path.exists(destination):
                        os.makedirs(destination)
                    else:
                        raise Exception("destination already exist")

                    for file in webdav.listdir(abs_path):
                        current = file.encode("ISO-8859-1").decode("UTF-8")
                        relative_root = os.path.relpath(current, source_path)

                        if webdav.isdir(file):
                            d = file.encode("ISO-8859-1").decode("UTF-8")
                            target_dir = os.path.join(target_path,
                                                      relative_root, d)
                            if not os.path.exists(target_dir):
                                os.makedirs(target_dir)
                            else:
                                raise Exception(
                                    "destination dir already exists")

                        if webdav.isfile(file):
                            f = file.encode("ISO-8859-1").decode("UTF-8")
                            source_file = os.path.join(current, f)
                            target_file_path = os.path.join(
                                target_path, relative_root)
                            target_file = os.path.join(target_path,
                                                       relative_root, f)
                            if not os.path.exists(target_file):
                                download_result = webdav.download(
                                    source_file, target_file_path)
                                if not download_result['success'] or len(
                                        download_result['file_list']
                                    ['failed']) > 0:
                                    raise download_result[
                                        'error'] if download_result[
                                            'error'] is not None else Exception(
                                                "Download error")
                            else:
                                raise Exception(
                                    "destination file already exists")

                elif webdav.isfile(abs_path):
                    try:
                        target_file = os.path.join(target_path, file_basename)
                        if not os.path.exists(target_file):
                            download_result = webdav.download(
                                abs_path, target_path)
                            if not download_result['success'] or len(
                                    download_result['file_list']
                                ['failed']) > 0:
                                raise download_result[
                                    'error'] if download_result[
                                        'error'] is not None else Exception(
                                            "Download error")
                        else:
                            raise Exception("destination file already exists")

                    except Exception as e:
                        self.logger.info("Cannot copy file %s , %s" %
                                         (abs_path, str(e)))
                        raise e

                success_paths.append(path)

            except Exception as e:
                self.logger.error("Error copy %s , error %s , %s" %
                                  (str(path), str(e), traceback.format_exc()))
                error_paths.append(path)

        return success_paths, error_paths
Beispiel #29
0
    def copy_files_to_tmp(self, target_path):
        if not os.path.exists(target_path):
            os.makedirs(target_path)

        webdav = WebDavConnection.create(self.login, self.session.get('server_id'), self.logger)

        success_paths = []
        error_paths = []

        for path in self.paths:
            try:
                abs_path = path
                source_path = webdav.parent(path)
                file_basename = webdav.path(abs_path)

                if webdav.isdir(abs_path):
                    destination = os.path.join(target_path, file_basename)

                    if not os.path.exists(destination):
                        os.makedirs(destination)
                    else:
                        raise Exception("destination already exist")

                    for file in webdav.listdir(abs_path):
                        current = file.encode("ISO-8859-1").decode("UTF-8")
                        relative_root = os.path.relpath(current, source_path)

                        if webdav.isdir(file):
                            d = file.encode("ISO-8859-1").decode("UTF-8")
                            target_dir = os.path.join(target_path, relative_root, d)
                            if not os.path.exists(target_dir):
                                os.makedirs(target_dir)
                            else:
                                raise Exception("destination dir already exists")

                        if webdav.isfile(file):
                            f = file.encode("ISO-8859-1").decode("UTF-8")
                            source_file = os.path.join(current, f)
                            target_file_path = os.path.join(target_path, relative_root)
                            target_file = os.path.join(target_path, relative_root, f)
                            if not os.path.exists(target_file):
                                download_result = webdav.download(source_file, target_file_path)
                                if not download_result['success'] or len(
                                        download_result['file_list']['failed']) > 0:
                                    raise download_result['error'] if download_result[
                                                                          'error'] is not None else Exception(
                                        "Download error")
                            else:
                                raise Exception("destination file already exists")

                elif webdav.isfile(abs_path):
                    try:
                        target_file = os.path.join(target_path, file_basename)
                        if not os.path.exists(target_file):
                            download_result = webdav.download(abs_path, target_path)
                            if not download_result['success'] or len(download_result['file_list']['failed']) > 0:
                                raise download_result['error'] if download_result[
                                                                      'error'] is not None else Exception(
                                    "Download error")
                        else:
                            raise Exception("destination file already exists")

                    except Exception as e:
                        self.logger.info("Cannot copy file %s , %s" % (abs_path, str(e)))
                        raise e

                success_paths.append(path)

            except Exception as e:
                self.logger.error(
                    "Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
                error_paths.append(path)

        return success_paths, error_paths
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            source_path = self.source.get('path')
            target_path = self.target.get('path')
            hash_str = self.random_hash()
            temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            self.logger.info("CopyFromSftpToWebDav process run source = %s , target = %s" % (source_path, target_path))

            target_webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
            t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
            t_total.start()

            for path in self.paths:
                try:
                    success_paths, error_paths = self.copy_files_to_tmp(temp_path)

                    if len(error_paths) == 0:
                        abs_path = self.get_abs_path(path)
                        file_basename = os.path.basename(abs_path)
                        uploading_path = temp_path + file_basename
                        if os.path.isdir(uploading_path):
                            uploading_path += '/'
                            file_basename += '/'

                        upload_result = target_webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
                                                             self.uploading_progress)

                        if upload_result['success']:
                            success_paths.append(path)
                            shutil.rmtree(temp_path, True)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            self.operation_progress["operation_done"] = True

            result = {
                "success": success_paths,
                "errors": error_paths
            }

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            operation_progress = {
                "total_done": False,
                "total": 0,
                "operation_done": False,
                "processed": 0
            }

            source_path = self.source.get('path')
            target_path = self.target.get('path')
            hash_str = self.random_hash()
            temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            self.logger.info("CopyFromWebDavToSFTP process run source = %s , target = %s" % (source_path, target_path))

            source_webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
            target_sftp = SFTPConnection.create(self.login, self.target.get('server_id'), self.logger)
            t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
            t_total.start()

            t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
            t_progress.start()

            for path in self.paths:
                try:
                    download_result = self.download_file_from_webdav(path, temp_path, source_webdav)

                    if download_result["success"]:
                        filedir = source_webdav.parent(path)
                        filename = path
                        if source_webdav.isdir(path):
                            filename = path + '/'
                        if filedir != '/':
                            filename = filename.replace(filedir, "", 1)
                        read_path = (temp_path + filename)
                        if not os.path.exists(read_path):
                            raise OSError("File not downloaded")

                        upload_success, upload_error = self.upload_files_recursive_to_sftp(
                            filename, temp_path, target_path, target_sftp, operation_progress)
                        if path in upload_success:
                            success_paths.append(path)
                            shutil.rmtree(temp_path, True)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            operation_progress["operation_done"] = True

            result = {
                "success": success_paths,
                "errors": error_paths
            }

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            source_path = self.source.get('path')
            target_path = self.target.get('path')
            hash_str = self.random_hash()
            temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            self.logger.info(
                "CopyBetweenWebDav process run source = %s , target = %s" %
                (source_path, target_path))

            source_webdav = WebDavConnection.create(
                self.login, self.source.get('server_id'), self.logger)
            target_webdav = WebDavConnection.create(
                self.login, self.target.get('server_id'), self.logger)
            t_total = threading.Thread(target=self.get_total,
                                       args=(self.operation_progress,
                                             self.paths))
            t_total.start()

            for path in self.paths:
                try:
                    download_result = self.download_file_from_webdav(
                        path, temp_path, source_webdav)

                    if download_result["success"]:
                        filedir = source_webdav.parent(path)
                        filename = path
                        if source_webdav.isdir(path):
                            filename = path + '/'
                        if filedir != '/':
                            filename = filename.replace(filedir, "", 1)
                        read_path = (temp_path + filename)
                        if not os.path.exists(read_path):
                            raise OSError("File not downloaded")

                        upload_result = self.upload_file_to_webdav(
                            read_path, target_path, filename, target_webdav)
                        if upload_result['success']:
                            success_paths.append(path)
                            shutil.rmtree(temp_path, True)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" %
                        (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            self.operation_progress["operation_done"] = True

            result = {"success": success_paths, "errors": error_paths}

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent':
                round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text':
                str(
                    int(
                        round(
                            float(len(success_paths)) / float(len(self.paths)),
                            2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id,
                            data=result,
                            progress=progress,
                            pid=self.pid,
                            pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id,
                          result,
                          pid=self.pid,
                          pname=self.name)
Beispiel #33
0
    def run(self):
        try:
            self.preload()
            abs_path = self.path
            self.logger.debug(
                "FM WebDav WriteFile worker run(), abs_path = %s" % abs_path)

            self.logger.debug("content %s" % pprint.pformat(self.content))
            self.logger.debug("encoding %s" % pprint.pformat(self.encoding))

            try:
                decoded = self.content.decode('utf-8')
                self.logger.debug("DECODED %s" % pprint.pformat(decoded))
            except Exception as e:
                self.logger.error("Error %s , %s" %
                                  (str(e), traceback.format_exc()))
                raise e

            webdav = WebDavConnection.create(self.login,
                                             self.session.get('server_id'),
                                             self.logger)

            try:
                hash_str = self.random_hash()
                temp_dir = os.path.abspath(TMP_DIR + '/' + self.login + '/' +
                                           hash_str + '/')
                os.makedirs(temp_dir)
                filedir = webdav.parent(self.path)
                filename = self.path
                if filedir != '/':
                    filename = filename.replace(filedir, "/", 1)
                temp_filename = temp_dir + filename
            except Exception as e:
                result = WebDavConnection.get_error(
                    e, "Unable to make file backup before saving \"%s\"." %
                    os.path.basename(abs_path))
                self.on_error(result)
                return

            try:
                content = decoded.encode(self.encoding)
                self.create_local_file(temp_filename, content)

                folder = webdav.parent(self.path)
                webdav.upload(temp_filename, folder, True)

                file_info = webdav.generate_file_info(self.path)

                shutil.rmtree(temp_dir, True)

                file_result = {"encoding": self.encoding, "item": file_info}

                result = {
                    "data": file_result,
                    "error": False,
                    "message": None,
                    "traceback": None
                }

                self.on_success(result)

            except Exception as e:
                try:
                    webdav.upload(
                        os.path.join(temp_dir, os.path.basename(abs_path)),
                        abs_path, True)
                except Exception as e:
                    result = WebDavConnection.get_error(
                        e,
                        "Unable to upload tmp file during write error \"%s\"."
                        % os.path.basename(abs_path))
                    self.on_error(result)
                    return

                result = WebDavConnection.get_error(
                    e, "Unable to write file \"%s\"." %
                    os.path.basename(abs_path))
                self.on_error(result)
                return

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(result)
Beispiel #34
0
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            operation_progress = {
                "total_done": False,
                "total": 0,
                "operation_done": False,
                "processed": 0
            }

            source_path = self.source.get('path')
            target_directory = self.target.get('path')

            if source_path is None:
                raise Exception("Source path empty")

            if target_directory is None:
                raise Exception("Target path empty")

            self.logger.info("MoveWebDav process run source = %s , target = %s" % (source_path, target_directory))

            webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
            t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
            t_total.start()

            # sleep for a while for better total counting
            time.sleep(REQUEST_DELAY)

            t_progress = threading.Thread(target=update_progress, args=(operation_progress,))
            t_progress.start()

            for path in self.paths:
                try:
                    replaced_path = path
                    if source_path != '/':
                        replaced_path = path.replace(webdav.parent(path), "/", 1)
                    if target_directory != '/':
                        target_path = target_directory + replaced_path
                    else:
                        target_path = replaced_path
                    
                    if webdav.isdir(path):
                        path += '/'

                    copy_result = webdav.move_file(path, webdav.path(target_path), overwrite=True)
                    if not copy_result['success'] or len(copy_result['file_list']['failed']) > 0:
                        raise copy_result['error'] if copy_result['error'] is not None else Exception(
                            "Upload error")
                    operation_progress["processed"] += 1

                    success_paths.append(path)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            operation_progress["operation_done"] = True

            result = {
                "success": success_paths,
                "errors": error_paths
            }

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
Beispiel #35
0
    def run(self):
        try:
            self.preload()
            success_paths = []
            error_paths = []

            source_path = self.source.get('path')
            target_path = self.target.get('path')

            if source_path is None:
                raise Exception("Source path empty")

            if target_path is None:
                raise Exception("Target path empty")

            source_path = self.get_abs_path(source_path)
            webdav = WebDavConnection.create(self.login,
                                             self.target.get('server_id'),
                                             self.logger)

            self.logger.debug(
                "CopyToWebDav process run source = %s , target = %s" %
                (source_path, target_path))

            t_total = threading.Thread(target=self.get_total,
                                       args=(self.operation_progress,
                                             self.paths))
            t_total.start()

            for path in self.paths:
                try:
                    abs_path = self.get_abs_path(path)
                    file_basename = os.path.basename(abs_path)

                    uploading_path = abs_path
                    if os.path.isdir(abs_path):
                        uploading_path += '/'
                        file_basename += '/'

                    result_upload = webdav.upload(uploading_path, target_path,
                                                  self.overwrite,
                                                  file_basename,
                                                  self.uploading_progress)

                    if result_upload['success']:
                        success_paths.append(path)

                except Exception as e:
                    self.logger.error(
                        "Error copy %s , error %s , %s" %
                        (str(path), str(e), traceback.format_exc()))
                    error_paths.append(path)

            self.operation_progress["operation_done"] = True

            result = {"success": success_paths, "errors": error_paths}

            # иначе пользователям кажется что скопировалось не полностью )
            progress = {
                'percent':
                round(float(len(success_paths)) / float(len(self.paths)), 2),
                'text':
                str(
                    int(
                        round(
                            float(len(success_paths)) / float(len(self.paths)),
                            2) * 100)) + '%'
            }
            time.sleep(REQUEST_DELAY)
            self.on_success(self.status_id,
                            data=result,
                            progress=progress,
                            pid=self.pid,
                            pname=self.name)

        except Exception as e:
            result = {
                "error": True,
                "message": str(e),
                "traceback": traceback.format_exc()
            }

            self.on_error(self.status_id,
                          result,
                          pid=self.pid,
                          pname=self.name)