def get_total(self,
progress_object,
paths,
count_dirs=True,
count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" %
(count_dirs, count_files))
sftp = SFTPConnection.create(self.login, self.source.get('server_id'),
self.logger)
for path in paths:
try:
abs_path = path
for current, dirs, files in sftp.walk(abs_path):
if count_files:
progress_object["total"] += len(files)
if sftp.isfile(abs_path):
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" %
(str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
sftp = SFTPConnection.create(self.login, self.source.get('server_id'), self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
sftp.rsync_from(path, target_path)
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def get_total(self, progress_object, paths, count_dirs=True, count_files=True):
self.logger.debug("start get_total() dirs = %s , files = %s" % (count_dirs, count_files))
sftp = SFTPConnection.create(self.login, self.source.get('server_id'), self.logger)
for path in paths:
try:
abs_path = path
for current, dirs, files in sftp.walk(abs_path):
if count_files:
progress_object["total"] += len(files)
if sftp.isfile(abs_path):
progress_object["total"] += 1
except Exception as e:
self.logger.error("Error get_total file %s , error %s" % (str(path), str(e)))
continue
progress_object["total_done"] = True
self.logger.debug("done get_total()")
return
def copy_files_to_tmp(self, target_path):
if not os.path.exists(target_path):
os.makedirs(target_path)
sftp = SFTPConnection.create(self.login, self.source.get('server_id'), self.logger)
success_paths = []
error_paths = []
for path in self.paths:
try:
sftp.rsync_from(path, target_path)
success_paths.append(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
return success_paths, error_paths
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("CopyFromWebDavToSFTP process run source = %s , target = %s" % (source_path, target_path))
source_webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
target_sftp = SFTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(operation_progress,))
t_progress.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_success, upload_error = self.upload_files_recursive_to_sftp(
filename, temp_path, target_path, target_sftp, operation_progress)
if path in upload_success:
success_paths.append(path)
shutil.rmtree(temp_path, True)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def get_sftp_connection(self, session):
return SFTPConnection.create(self.login, session['server_id'], self.logger)
def get_sftp_connection(self, session):
return SFTPConnection.create(self.login, session['server_id'],
self.logger)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
operation_progress = {
"total_done": False,
"total": 0,
"operation_done": False,
"processed": 0
}
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("MoveFromWebDavToSFTP process run source = %s , target = %s" % (source_path, target_path))
source_webdav = WebDavConnection.create(self.login, self.source.get('server_id'), self.logger)
target_sftp = SFTPConnection.create(self.login, self.target.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(operation_progress, self.paths))
t_total.start()
t_progress = threading.Thread(target=self.update_progress, args=(self, operation_progress,))
t_progress.start()
for path in self.paths:
try:
download_result = self.download_file_from_webdav(path, temp_path, source_webdav)
if download_result["success"]:
filedir = source_webdav.parent(path)
filename = path
if source_webdav.isdir(path):
filename = path + '/'
if filedir != '/':
filename = filename.replace(filedir, "", 1)
read_path = (temp_path + filename)
if not os.path.exists(read_path):
raise OSError("File not downloaded")
upload_success, upload_error = self.upload_files_recursive_to_sftp(
filename, temp_path, target_path, target_sftp, operation_progress)
if len(upload_error) == 0:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_webdav.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info("MoveFromSftpToWebDav process run source = %s , target = %s" % (source_path, target_path))
target_webdav = WebDavConnection.create(self.login, self.target.get('server_id'), self.logger)
source_sftp = SFTPConnection.create(self.login, self.source.get('server_id'), self.logger)
t_total = threading.Thread(target=self.get_total, args=(self.operation_progress, self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(uploading_path, target_path, self.overwrite, file_basename,
self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_sftp.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" % (str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {
"success": success_paths,
"errors": error_paths
}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent': round(float(len(success_paths)) / float(len(self.paths)), 2),
'text': str(int(round(float(len(success_paths)) / float(len(self.paths)), 2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id, data=result, progress=progress, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
success_paths = []
error_paths = []
source_path = self.source.get('path')
target_path = self.target.get('path')
hash_str = self.random_hash()
temp_path = TMP_DIR + '/' + self.login + '/' + hash_str + '/'
if source_path is None:
raise Exception("Source path empty")
if target_path is None:
raise Exception("Target path empty")
self.logger.info(
"MoveFromSftpToWebDav process run source = %s , target = %s" %
(source_path, target_path))
target_webdav = WebDavConnection.create(
self.login, self.target.get('server_id'), self.logger)
source_sftp = SFTPConnection.create(self.login,
self.source.get('server_id'),
self.logger)
t_total = threading.Thread(target=self.get_total,
args=(self.operation_progress,
self.paths))
t_total.start()
for path in self.paths:
try:
success_paths, error_paths = self.copy_files_to_tmp(
temp_path)
if len(error_paths) == 0:
abs_path = self.get_abs_path(path)
file_basename = os.path.basename(abs_path)
uploading_path = temp_path + file_basename
if os.path.isdir(uploading_path):
uploading_path += '/'
file_basename += '/'
upload_result = target_webdav.upload(
uploading_path, target_path, self.overwrite,
file_basename, self.uploading_progress)
if upload_result['success']:
success_paths.append(path)
shutil.rmtree(temp_path, True)
source_sftp.remove(path)
except Exception as e:
self.logger.error(
"Error copy %s , error %s , %s" %
(str(path), str(e), traceback.format_exc()))
error_paths.append(path)
self.operation_progress["operation_done"] = True
result = {"success": success_paths, "errors": error_paths}
# иначе пользователям кажется что скопировалось не полностью )
progress = {
'percent':
round(float(len(success_paths)) / float(len(self.paths)), 2),
'text':
str(
int(
round(
float(len(success_paths)) / float(len(self.paths)),
2) * 100)) + '%'
}
time.sleep(REQUEST_DELAY)
self.on_success(self.status_id,
data=result,
progress=progress,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
