class Upload():
def __init__(self):
readxml_obj = XmlReading()
self.ip = readxml_obj.get_cluster_ip()
self.src_path = readxml_obj.get_src_path()
self.dst_path = readxml_obj.get_dst_path()
self.tar_type = readxml_obj.get_tar_type()
self.database_obj = DatabaseInterface()
self.tar_obj = Tar()
# |----------------------------------------------------------------------------|
# get_folder_size
# |----------------------------------------------------------------------------|
def get_folder_size(self, folder_path):
folder_size_content = subprocess.Popen(["du -sh {}".format(folder_path)],
stdout=subprocess.PIPE, shell=True)
content_list = str(folder_size_content.stdout.read())
content_list = re.findall(r'([-+]?\d*\.\d+|\d+)(\w+)', content_list)
#print("content_list : ", content_list)
first_tuple = content_list[0]
folder_size = float(first_tuple[0])
if "M" in first_tuple:
folder_size = folder_size / 1000
if "K" in first_tuple:
folder_size = folder_size / (1000 * 1000)
print(folder_size)
return folder_size
# |----------------------End of get_folder_size-------------------------------|
# |----------------------------------------------------------------------------|
# update_status_to_node
# |----------------------------------------------------------------------------|
def update_status_to_node(self,slide_name,slide_metadata,activity_status,error_info):
basket_status_url = "http://10.20.0.1:1337/basketslot/status/"
slide_metadata['data'] = {}
slide_metadata['data']['slide_name'] = slide_name
slide_metadata['data']['activity_status'] = activity_status
slide_metadata['data']['error_info'] = error_info
#print(slide_metadata)
try:
resp = requests.post(url=basket_status_url,
data=json.dumps(slide_metadata),
timeout=20)
if resp.status_code == 200:
ServiceLogger.get().log_info("Request to node for "
"folder name {} "
"succeeded.".
format(slide_name))
else:
ServiceLogger.get().log_info("Request to node for "
"folder name {} "
"failed with response code: {}.".
format(slide_name,resp.status_code))
self.database_obj.log(slide_name,slide_metadata,basket_status_url,resp.status_code,'node')
except Exception as msg:
ServiceLogger.get().log_error("Request to node for "
"folder name {} failed due to: {} ".
format( slide_name, msg))
self.database_obj.log(slide_name,slide_metadata,basket_status_url,'500','node')
# |----------------------End of update_status_to_node-------------------------|
# |----------------------------------------------------------------------------|
# tar_slide_folder
# |----------------------------------------------------------------------------|
def tar_slide(self,slide_path,slide_name):
'''
print('slide_path ',slide_path)
os.chdir(slide_path)
tarcmd = "tar -cf {}.tar {}".format(slide_name,slide_name)
print(tarcmd)
status = os.system(tarcmd)
'''
status = 123
if self.tar_type == '1':
status = self.tar_obj.tar_slide_folder(slide_path,slide_name)
elif self.tar_type == '2':
status = self.tar_obj.tar_complete_slide_folder(slide_path,slide_name)
elif self.tar_type == '3':
status = self.tar_obj.tar_grid_folder(slide_path,slide_name)
return status
# |----------------------End of tar_slide_folder-------------------------|
# |----------------------------------------------------------------------------|
# transfer_slide_folder_to_cluster
# |----------------------------------------------------------------------------|
def transfer_slide_folder_to_cluster(self, slide_name,metadata):
move_status = False
err_msg = ''
status = 123
#slide_path =''
try:
slide_path = join(self.src_path,slide_name)
if exists(slide_path) is True:
ServiceLogger.get().log_info("*** CALL for tar ***")
self. update_status_to_node(slide_name,metadata,'transfering','')
tar_status = self.tar_slide(self.src_path,slide_name)
print("Move src_path: ", slide_path, " to: ", self.dst_path)
if tar_status == 0:
command = "sshpass -p 'adminspin#123' rsync -e 'ssh -o \
StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' "\
"-av {} adminspin@{}:{}".format(slide_path, self.ip, self.dst_path)
#print("command: ",command)
ServiceLogger.get().log_info("rsync command: {}".
format(command))
startTime = time.time()
status = os.system(command)
endTime = time.time()
total_time = endTime - startTime
folder_size = self.get_folder_size(slide_path)
print("Total time to move {} of size {}GB is: {}".format(slide_path,
folder_size,
total_time))
ServiceLogger.get().log_info("Total time to move {} of size {}GB is: {}".
format(slide_path,folder_size,total_time))
if status == 0:
move_status = True
rmcmd = "rm -rf {} {}".format(slide_path,slide_path)
#print("rmcmd ",rmcmd)
ServiceLogger.get().log_info("rsync rmcmd: {}".
format(rmcmd))
os.system(rmcmd)
else:
move_status = False
print('Fail to transfer')
ServiceLogger.get().log_error("Fail to transfer {}".
format(slide_name))
self. update_status_to_node(slide_name,metadata,'error','Fail to transfer the data')
else:
self. update_status_to_node(slide_name,metadata,'error','Fail to tar the data')
else:
err_msg ='Yes'
print("Slide does not exits")
ServiceLogger.get().log_error("Slide: {} does not exits".
format(slide_name))
except Exception as msg:
ServiceLogger.get().log_error("exception during transfer slide :{} "
"failed due to: {}".
format(slide_name, msg))
return move_status,err_msg
class Sender():
def __init__(self):
self.joblist = []
self.upload_obj = Upload()
self.dcider_flag = True
self.database_obj = DatabaseInterface()
# |----------------------------------------------------------------------------|
# job_queue
# |----------------------------------------------------------------------------|
def job_queue(self, slide, metadata):
print('job_queue ', slide)
data = {"slide_name": slide, "metadata": metadata}
self.joblist.append(data)
#print(len(self.joblist))
self.database_obj.log(slide, metadata,
'http://localhost:8025/transfer/', '200',
'transfer')
if self.dcider_flag is True:
self.dcider_flag = False
self.decider()
# |----------------------End of job_queue-------------------------------|
# |----------------------------------------------------------------------------|
# call_restoration_service
# |----------------------------------------------------------------------------|
def call_restoration_service(self, slide_folder_name, slide_metadata):
restoration_service_url = "http://10.20.0.1:8026/restore/{}".\
format(slide_folder_name)
print(restoration_service_url)
print(slide_metadata)
try:
resp = requests.post(url=restoration_service_url,
data=json.dumps(slide_metadata),
timeout=5)
if resp.status_code == 200:
ServiceLogger.get().log_info(
"Request to restore service for "
"slide {} "
"succeeded.".format(slide_folder_name))
else:
ServiceLogger.get().log_info(
"Request to restore service for "
"slide {}"
"failed with response code: {}.".format(
slide_folder_name, resp.status_code))
self.database_obj.log(slide_folder_name, slide_metadata,
restoration_service_url, resp.status_code,
'restore')
except Exception as msg:
print("Request failed due to: ", msg)
ServiceLogger.get().log_error(
"Request to transfer service for slide "
"{} failed due to: {}".format(slide_folder_name, msg))
self.database_obj.log(slide_folder_name, slide_metadata,
restoration_service_url, '500', 'restore')
# |----------------------End of call_restoration_service-------------------------|
# |----------------------------------------------------------------------------|
# decider
# |----------------------------------------------------------------------------|
def decider(self):
if (len(self.joblist) > 0):
slide_name = self.joblist[0]["slide_name"]
metadata = copy.deepcopy(self.joblist[0]["metadata"])
status, error = self.upload_obj.transfer_slide_folder_to_cluster(
slide_name, metadata)
#time.sleep(10)
print(status)
#if status is True or error == 'Yes':
if status is True:
print("----------Finish-------------------")
self.call_restoration_service(slide_name,
self.joblist[0]["metadata"])
self.joblist.pop(0)
self.decider()
else:
self.dcider_flag = True