def fetch_locks(self, rec, **kwargs):
file_created = []
remote_source_location = rec.get("remote_source_location")
result = runCommand("find {}".format(
remote_source_location), True, None, **kwargs)
if result and result.return_code == 0:
result = None
result = runCommand(
"cd {}; ls -tar |grep .lock".format(remote_source_location), True, None, **kwargs)
if not result or result.return_code != 0:
raise Exception(" unable to find locks with unknown error")
else:
for line in result.stdout.split():
try:
self.handle_lock(rec, line, **kwargs)
file_created.append(
"Status: Success File: " + remote_source_location + "/" + line)
print "Status: Success File: " + remote_source_location + "/" + line
except Exception as exp:
print "Unable to process lock:" + line + " Error:" + str(exp)
file_created.append(
"Status: Failed File: " + remote_source_location + "/" + line + " Error:" + str(exp))
else:
raise Exception("Unable to access Path: " + remote_source_location)
return file_created
def validate_if_everthing_was_copied(self, rec, dir_name, no_of_files_in_remote, no_of_files_required, no_of_files_in_local, **kwargs):
remote_source_location = rec.get(
"remote_source_location") + "/" + dir_name
current_import_path = self.current_import_path + "/" + dir_name
if no_of_files_in_remote <> no_of_files_required or no_of_files_in_local <> no_of_files_required:
self.write_wip(current_import_path, no_of_files_required,
no_of_files_in_remote, no_of_files_in_local)
else:
os.remove(current_import_path + "/wip")
runCommand("rm -f " + remote_source_location +
".lock", False, None, **kwargs)
def handle_file(self, rec, file_name, no_of_files_required, **kwargs):
file_to_copy = self.get_md5sum_checksum(
rec, file_name, False, **kwargs)
current_import_file = self.current_import_path + \
"/" + file_to_copy[0].split()[1]
remote_source_file = rec.get(
"remote_source_location") + "/" + file_to_copy[0].split()[1]
self.copy_file_to_remote(rec, remote_source_file,
file_to_copy[0].split()[0], current_import_file)
if "wip" not in current_import_file and self.verify_md5sum_in_local(file_to_copy[0].split()[0], current_import_file):
runCommand("rm -f " + remote_source_file +
".lock", False, None, **kwargs)
def do_push(rec, file_num, file_name, source_file, folder_to_push_to,
**kwargs):
syncRequestdb.update_step_status(rec["_id"], file_name, file_num,
"processing")
copyToRemote(source_file, folder_to_push_to + "/" + file_name + ".wip",
**kwargs)
runCommand(
"mv " + folder_to_push_to + "/" + file_name + ".wip " +
folder_to_push_to + "/" + file_name, False, None, **kwargs)
try:
runCommand("chmod -R 777 " + folder_to_push_to, False, None, **kwargs)
except Exception: # catch *all* exceptions
traceback.print_exc()
print '##### YOU CAN IGNORE THIS ERROR ####'
syncRequestdb.update_step_status(rec["_id"], file_name, file_num, "Done")
def clear_old_data(rec, **kwargs):
# ANY OLD ZIP ?.LETS REMOVE THEM
remove_old_file(rec)
# MOVE ONY IF ITS NEW SYNC
if rec.get("zip_file_name") and str(
rec.get("zip_file_name")).strip() <> "":
try:
runCommand(
"rm -rf " + get_folder_to_push_to(rec) + "/" +
rec["zip_file_name"], True, None, **kwargs)
except Exception as exception:
print "Push Service :Failed to move files to ../SyncOld. Error: " + str(
exception)
clear_sync_data(rec)
update_request_in_database(rec)
def handle_dir(self, rec, dir_name, no_of_files_required, **kwargs):
remote_source_location = rec.get(
"remote_source_location") + "/" + dir_name
current_import_path = self.current_import_path + "/" + dir_name
result = runCommand("find {}".format(
remote_source_location), True, None, **kwargs)
if result and result.return_code == 0:
data = self.get_md5sum_checksum(rec, dir_name, True, **kwargs)
if len(data) == 0:
raise ValueError(
"No files found in: " + remote_source_location + " with name not having .wip")
else:
self.mkdir_p(current_import_path)
futures = []
pool = ThreadPoolExecutor(int(self.count_of_files),__name__+".handle_dir")
self.write_wip(current_import_path, no_of_files_required, str(len(data)), str(
len([f for f in os.listdir(current_import_path) if str(f).lower() <> "wip"])))
for file_to_copy in data:
futures.append(pool.submit(self.copy_file_to_remote, rec, remote_source_location + "/" + file_to_copy.split()[
1], file_to_copy.split()[0], current_import_path + "/" + file_to_copy.split()[1]))
wait(futures)
exceptions_list = []
for future in futures:
if future.exception():
exceptions_list.append(str(future.exception()))
if len(exceptions_list):
raise Exception(','.join(exceptions_list))
self.write_wip(current_import_path, no_of_files_required, str(len(data)), str(
len([f for f in os.listdir(current_import_path) if str(f).lower() <> "wip"])))
self.validate_if_everthing_was_copied(rec, dir_name, len(data), no_of_files_required, len(
[f for f in os.listdir(current_import_path) if "wip" not in str(f).lower()]), **kwargs)
else:
raise Exception(
"Path:" + remote_source_location + " does not exists")
def handle_lock(self, rec, lock_name, **kwargs):
remote_source_location = rec.get("remote_source_location")
result = None
result = runCommand("cd {}; cat ".format(
remote_source_location) + lock_name, True, None, **kwargs)
if not result:
raise Exception(" unable to read lock " + lock_name +
" reason is unknown.Please check logs")
if result.return_code != 0:
raise Exception(" unable to read lock with unknown error")
else:
data = result.stdout.split(",")
try:
if len(data) <> 2:
raise ValueError()
no_of_files_required = int(data[0])
except Exception:
raise ValueError("The lock file: " + lock_name + " is invalid")
self.syncRequestdb.update_deployment_request_status(str(
rec["_id"]), "Pulling Started", "Trying to pull: " + remote_source_location + "/" + str(lock_name.split(".lock")[0]))
if no_of_files_required > 1:
self.handle_dir(rec, lock_name.split(".lock")[
0], no_of_files_required, **kwargs)
else:
self.handle_file(rec, lock_name.split(".lock")[
0], no_of_files_required, **kwargs)
self.syncRequestdb.update_deployment_request_status(str(
rec["_id"]), "Pulling Started", "Pulled: " + remote_source_location + "/" + str(lock_name.split(".lock")[0]))
def get_md5sum_checksum(self, rec, name, is_dir_ind=False, **kwargs):
remote_source_location = rec.get("remote_source_location")
if is_dir_ind:
remote_source_location = remote_source_location + "/" + name
result = None
if is_dir_ind:
result = runCommand("cd {}; ls * -1 | sort |md5sum * | grep -v .wip".format(
remote_source_location), True, None, **kwargs)
else:
result = runCommand("cd {}; md5sum ".format(
remote_source_location) + name, True, None, **kwargs)
if not result:
raise Exception("Path: " + remote_source_location +
" seems to be invalid/empty")
if result.return_code != 0:
raise Exception(" unable to md5sum on :" + name + " with unknown error")
else:
return [x for x in result.stdout.split('\n') if x]
def start_push_to_remote(rec, **kwargs):
folder_to_push_to = PushHelperService.get_folder_to_push_to(rec)
create_lock(rec, "WIP", **kwargs)
if (os.path.isfile(rec["file_created"])):
start_thread(rec, folder_to_push_to)
else:
folder_to_push_to = (
folder_to_push_to + "/" +
os.path.basename(rec["file_created"])).split(".")[0]
createFolder(folder_to_push_to, **kwargs)
file_count_to_push = 0
file_count_of_pushed = 0
file_remaining_to_push = 0
if rec.get("file_list"):
for f in rec.get("file_list").keys():
file_count_to_push = file_count_to_push + 1
if rec.get("file_list").get(f)["file_status"] not in [
"new", "processing"
]:
file_count_of_pushed = file_count_of_pushed + 1
else:
file_remaining_to_push = file_remaining_to_push + 1
runCommand(
"echo Last locked by push at $(date) file_count_to_push: " +
str(file_count_to_push) + " file_count_of_pushed: " +
str(file_count_of_pushed) + " file_remaining_to_push: " +
str(file_remaining_to_push) + " >" + folder_to_push_to + "/wip",
False, None, **kwargs)
start_thread(rec, folder_to_push_to)
runCommand("rm -f " + folder_to_push_to + "/wip", False, None,
**kwargs)
create_lock(rec, "DONE", **kwargs)
copy_rec = {}
copy_rec["_id"] = str(rec["_id"])
copy_rec["copy_in_progress"] = "false"
copy_rec["last_success_push_date"] = datetime.now()
PushHelperService.update_request_in_database(copy_rec)
def do_copy(self, rec, remote_full_path, source_md5, file_to_be_copied_full_path, **kwargs):
try:
result = runCommand("find {}".format(
remote_full_path), True, None, **kwargs)
if result and result.return_code == 0:
if "wip" not in file_to_be_copied_full_path.lower() and not self.verify_md5sum_in_local(source_md5, file_to_be_copied_full_path, **kwargs):
copyFromRemote(
remote_full_path, file_to_be_copied_full_path + ".wip", **kwargs)
if os.path.exists(file_to_be_copied_full_path):
os.remove(file_to_be_copied_full_path)
os.rename(file_to_be_copied_full_path +
".wip", file_to_be_copied_full_path)
else:
raise ValueError(
"File:" + remote_full_path + " does not exists")
except Exception as exp:
try:
if os.path.exists(file_to_be_copied_full_path + ".wip"):
os.remove(file_to_be_copied_full_path + ".wip")
if os.path.exists(file_to_be_copied_full_path):
os.remove(file_to_be_copied_full_path)
except Exception:
pass
raise exp
def pingMachine(self, rec, **kwargs):
try:
runCommand("hostname", False, **kwargs)
except Exception as e:
raise ValueError('ping machine failed with error : ' + str(e))