def import_task_log(self):
self.doingWork.emit(True, "Importing task log", self.thread_name)
proj_path = self.DUG_proj_path
remote_path = posixpath.join(proj_path, 'register', 'active_tasks')
remote_lock_path = posixpath.join(proj_path, 'register',
'app_task_sync_lock')
local_path = os.path.join(os.getcwd(), 'temp', 'active_tasks')
if os.path.exists(local_path):
try:
os.remove(local_path)
except Exception as error:
logger.error("Exception: Local task_log_busy waiting 7s")
logger.error(error)
self.doingWork.emit(
True, "Exception: Local task_log_busy waiting 7s",
self.thread_name)
time.sleep(7)
self.import_task_log()
status_lock = check_generic_path(self.DUG_connection_obj,
remote_lock_path)
if status_lock == 'True':
logger.warning(
"Deliverables QC daemon is writing to file, wait 5s")
self.doingWork.emit(
True, "Deliverables QC daemon is writing to file, wait 5s",
self.thread_name)
time.sleep(5)
self.import_task_log()
else:
status = check_generic_path(self.DUG_connection_obj, remote_path)
if status == 'True':
# Now FTP the file
logger.info(
"Now copying over the task log from remote host ..")
self.doingWork.emit(
True, "Now copying over the task log from remote host ..",
self.thread_name)
try:
self.DUG_connection_obj.sftp_client.get(
remote_path, local_path)
logger.info('Done ..')
self.local_path = local_path
self.extract_task_info()
except Exception as error:
logger.error("Exception: Unable to copy to local host ")
logger.error(error)
self.doingWork.emit(
True, "Exception: Unable to copy to local host ",
self.thread_name)
else:
logger.info("No active tasks on remote host..")
self.doingWork.emit(True, "No active tasks on remote host..",
self.thread_name)
def segy_qc_from_form(self, attribute):
(bin_def_path, trc_def_path) = self.create_def_files_and_sFTP()
file_path = posixpath.join(self.dir_service.data_dir_path_dict['data'],
attribute)
logger.info("Now running Qc script on: " + file_path)
log_name = str(
self.Deliverable.id
) + "_" + self.Deliverable.name + "_" + attribute + '.headerlog'
log_dir = self.dir_service.data_dir_path_dict['headers']
log_path = posixpath.join(log_dir, log_name)
self.qc_log_path = log_path
# if log exists before delete it and than redo it
status = check_generic_path(self.DUG_connection_obj, log_path)
script_name = self.DUG_connection_obj.DUG_project_name + "_segy-hcm"
script_path = posixpath.join("/d/home/share/bin", script_name)
cmd = str(
"nohup " + script_path + " " + file_path + " -bd " + bin_def_path +
" -td " + trc_def_path + " -l " + log_path + " > /dev/null 2>&1 &"
) # change this line to modify the command and the python package needs to be changed as well
if status == 'True':
message = "The log already exists either the QC is running or you are trying to run it again. Please check if the job is still running or if finished press ok to remove old file and create a new one !!"
self.pop_up_message_box = pop_up_message_box(message, 'Warning')
self.pop_up_message_box.closed.connect(
lambda: self.run_segy_qc(cmd, True))
self.pop_up_message_box.show()
# modify the existing Db entry
db_entry = self.db_connection_obj.sess.query(
self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.
segy_on_disk_qc_log_path == log_path).first()
db_entry.segy_on_disk_qc_run_status = True
db_entry.segy_on_disk_qc_status = None
db_entry.segy_on_disk_qc_approver_name = None
db_entry.segy_on_disk_qc_approval_time = None
self.db_connection_obj.sess.commit()
else:
# execute the command on the DUG workstation
self.run_segy_qc(cmd, False)
db_entry = self.db_connection_obj.sess.query(
self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.
segy_on_disk_file_path == file_path).first()
db_entry.segy_on_disk_qc_run_status = True
db_entry.segy_on_disk_qc_log_path = log_path
self.db_connection_obj.sess.commit()
def create_large_files_project_dir(self):
logger.info(
"Now checking if the SEGY file dir for project exists in ../large_files"
)
large_files_root_path = large_file_root_dict[self.use_location]
project_name = self.db_connection_obj.db_name.split("_")[1]
self.large_file_dir_proj_path = posixpath.join(large_files_root_path,
project_name)
status = check_generic_path(self.DUG_connection_obj,
self.large_file_dir_proj_path)
if status == 'True':
logger.info(self.large_file_dir_proj_path +
" :The directory already exists..")
else:
logger.info("Now creating ::" + self.large_file_dir_proj_path)
create_generic_directory(self.DUG_connection_obj,
self.large_file_dir_proj_path)
self.DUG_connection_obj.large_files_dir = self.large_file_dir_proj_path
def show_export_upate_pop_up(self):
sender = self.sender()
self.obj_name = str(sender.objectName())
# first check if the file exists in the directory
#get the file path for the deliverable id data dir from the database
file_name = str(self.obj_name + ".sgy")
file_path = posixpath.join(self.dir_path,file_name)
status = check_generic_path(self.DUG_connection_obj,file_path)
if status == 'True':
message = str("Found the file, Update the status for SEGY export for: " + file_path)
self.approval_pop_up = pop_up_approval_box(message)
self.approval_pop_up.closed.connect(self.update_segy_export)
self.approval_pop_up.setMinimumWidth(400)
self.approval_pop_up.show()
else:
message = str("Unable to find the file for: " + file_path)
self.approval_pop_up = pop_up_message_box(self,message,'Critical')
self.approval_pop_up.setMinimumWidth(300)
self.approval_pop_up.show()
def add_data(self):
self.data_dir_entry = self.db_connection_obj.sess.query(self.db_connection_obj.Deliverables_data_dir).filter(
self.db_connection_obj.Deliverables_data_dir.deliverable_id == self.deliverable_id).filter(
self.db_connection_obj.Deliverables_data_dir.dir_type == 'data').first()
self.dir_path = self.data_dir_entry.path
sgyt_data = self.db_connection_obj.sess.query(self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.deliverable_id == self.deliverable_id).first()
self.data_dict = {self.deliverable.name : sgyt_data}
self.grid.addWidget(create_center_data(self.deliverable.name), 1, 0)
self.grid.addWidget(create_center_data(self.deliverable.name), 1, 13)
self.grid.addWidget(decide_and_create_label(sgyt_data.sgyt_approval_status),1,1)
if sgyt_data.sgyt_approval_status: # if SGYT approval status is not True do not proceed
pb_segy_exp = QtGui.QPushButton("Update")
pb_segy_exp.setObjectName(self.deliverable.name)
pb_segy_exp.clicked.connect(self.show_export_upate_pop_up)
self.grid.addWidget(pb_segy_exp,1,2)
file_name = str(self.deliverable.name + '.sgy')
file_path = posixpath.join(self.dir_path, file_name)
status = check_generic_path(self.DUG_connection_obj, file_path)
if status == 'True': # if the SEGYT file is on disk writing or finished, otherwise STOP
self.grid.addWidget(decide_and_create_label(sgyt_data.segy_disk_export_status), 1, 3)
if sgyt_data.segy_disk_export_status is not None: # if the export status is set to true or false add exporter name and time stamp labels
self.grid.addWidget(create_center_data(sgyt_data.segy_exporter_name),1,4)
self.grid.addWidget(create_center_data(sgyt_data.segy_on_disk_time_stamp),1,5)
if sgyt_data.segy_disk_export_status: # Add the option to QC the file if the export is set to True
self.grid.addWidget(decide_and_create_label(sgyt_data.segy_on_disk_qc_run_status),1,6) # The run status should be update by SEGY on disk QC ops
if sgyt_data.segy_on_disk_qc_run_status: # if the run status is True add the button to approve log and link to the log,extractionstatus
pb_connect_log = QtGui.QPushButton('View QC log')
pb_connect_log.setObjectName(sgyt_data.segy_on_disk_qc_log_path)
pb_connect_log.clicked.connect(self.show_QC_log)
self.grid.addWidget(pb_connect_log,1,7)
self.grid.addWidget(decide_and_create_label(sgyt_data.header_extraction_flag),1,8)
if sgyt_data.header_extraction_flag: # if the extraction status is true, add approve button and qc status button
pb_approve_log = QtGui.QPushButton('Approve')
pb_approve_log.setObjectName(sgyt_data.segy_on_disk_qc_log_path)
pb_approve_log.clicked.connect(self.show_approve_qc_log)
self.grid.addWidget(pb_approve_log,1,9)
self.grid.addWidget(decide_and_create_label(sgyt_data.segy_on_disk_qc_status),1,10)
if sgyt_data.segy_on_disk_qc_status is not None: # Now add the time and name for approver
self.grid.addWidget(create_center_data(sgyt_data.segy_on_disk_qc_approver_name),1,11)
self.grid.addWidget(create_center_data(sgyt_data.segy_on_disk_qc_approval_time_stamp), 1, 12)
def check_path(self, path_to_check):
status = check_generic_path(self.DUG_connection_obj, path_to_check)
return status
def create_SEGY_3D_sgyt(self):
result = self.db_connection_obj.sess.query(
self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.deliverable_id ==
self.Deliverable.id).first(
) # This time we do not need a sequence in the filter mocde
if result is None:
sgyt_file_name = create_3D_sgyt(self.Deliverable, self.IL_range,
self.XL_range, int(self.reel))
user_file_name = "user_" + sgyt_file_name
# SFTP the template to the DUG workstation
# 1. check if the file already exists on the DUG workstation
self.dir_service.set_deliverable(self.Deliverable)
dir_for_checking = self.dir_service.data_dir_path_dict['masters']
local_path = os.path.join(os.getcwd(), 'temp', sgyt_file_name)
remote_path = posixpath.join(dir_for_checking, sgyt_file_name)
status = check_generic_path(self.DUG_connection_obj, remote_path)
logger.info(
"Now attempting to transfer the file to the DUG workstation..."
)
if status == 'True':
action = get_item_through_dialogue(
self.parent,
'Now attempting to transfer the file to the DUG workstation... type y to continue, n to exit'
)
if action == 'y':
SFTP_generic_file(self.DUG_connection_obj, local_path,
remote_path)
else:
logger.warning('Aborting the file transfer!!!!')
else:
SFTP_generic_file(self.DUG_connection_obj, local_path,
remote_path)
# now create a new DAO object
new_obj = self.db_connection_obj.SEGY_QC_on_disk()
new_obj.line_name = '3D deliverable'
new_obj.deliverable_id = self.Deliverable.id
new_obj.sgyt_status = True
new_obj.sgyt_reel_no = self.Deliverable.reel_prefix + str(
self.reel)
new_obj.sgyt_min_il = self.IL_range[0]
new_obj.sgyt_max_il = self.IL_range[1]
new_obj.sgyt_min_xl = self.XL_range[0]
new_obj.sgyt_max_xl = self.XL_range[1]
new_obj.sgyt_fgsp = None
new_obj.sgyt_lgsp = None
new_obj.sgyt_min_ffid = None
new_obj.sgyt_max_ffid = None
new_obj.sgyt_user_path = posixpath.join(dir_for_checking,
user_file_name)
new_obj.sgyt_unix_path = remote_path
new_obj.sgyt_exp_uname = self.username
new_obj.sgyt_time_stamp = datetime.datetime.now().strftime(
"%I:%M%p on %B %d, %Y")
self.db_connection_obj.sess.add(new_obj)
self.db_connection_obj.sess.commit()
logger.info(
"The new object for SEGY on disk QC SEGY template export is now added to the database..."
)
else:
message = str("The SGYT file for deliverable_id : " +
str(self.Deliverable.id) + ": name : " +
self.Deliverable.name + " was exported by : " +
result.sgyt_exp_uname + ' on : ' +
result.sgyt_time_stamp +
" Enter reason to re-export: ")
perform = change_log_creation(gui=self.parent,
conn_obj=self.db_connection_obj,
message=message,
type_entry="change",
location='sgyt')
if perform:
sgyt_file_name = create_3D_sgyt(self.Deliverable,
self.IL_range, self.XL_range,
self.reel)
user_file_name = "user_" + sgyt_file_name
# SFTP the template to the DUG workstation
# 1. check if the file already exists on the DUG workstation
self.dir_service.set_deliverable(self.Deliverable)
dir_for_checking = self.dir_service.data_dir_path_dict[
'masters']
local_path = os.path.join(os.getcwd(), 'temp', sgyt_file_name)
remote_path = posixpath.join(dir_for_checking, sgyt_file_name)
status = check_generic_path(self.DUG_connection_obj,
remote_path)
logger.info(
"Now attempting to transfer the file to the DUG workstation..."
)
if status == 'True':
message = "File already exists on DUG system, type y to continue, n to exit"
action = get_item_through_dialogue(self.parent, message)
if action == 'y':
SFTP_generic_file(self.DUG_connection_obj, local_path,
remote_path)
else:
logger.warning('Aborting the file transfer!!!!')
else:
SFTP_generic_file(self.DUG_connection_obj, local_path,
remote_path)
print "File transfer complete .. "
# now create a new DAO object
result.sgyt_reel_no = self.Deliverable.reel_prefix + str(
self.reel)
result.sgyt_fgsp = None
result.sgyt_lgsp = None
result.sgyt_min_ffid = None
result.sgyt_max_ffid = None
result.sgyt_min_il = self.IL_range[0]
result.sgyt_max_il = self.IL_range[1]
result.sgyt_min_xl = self.XL_range[0]
result.sgyt_max_xl = self.XL_range[1]
result.sgyt_user_path = posixpath.join(dir_for_checking,
user_file_name)
result.sgyt_unix_path = remote_path
result.sgyt_exp_uname = self.username
result.sgyt_time_stamp = datetime.datetime.now().strftime(
"%I:%M%p on %B %d, %Y")
if result.sgyt_approval_status is not None:
# this step clears the previous status flag as this is a new file and should be checked again
result.sgyt_approval_status = None
result.sgyt_approver_name = None
result.sgyt_approval_time = None
self.db_connection_obj.sess.commit()
def set_attribute(self, attribute, caller, ops):
if ops == 'SGYT':
if caller == 'Deliverable':
self.Deliverable = self.disp_name_dict[attribute]
#Check if the deliverable is of the type sequence wise
if self.Deliverable.type in sequence_wise_SEGY:
#Now check if DUG SGYT master is defined for the deliverable
if self.Deliverable.sgyt_master_status:
#print self.Deliverable.sgyt.decode('base64')
if self.Deliverable.media in multiple_per_tape_list:
self.choose_sequences(ops)
else:
self.choose_sequence(ops)
else:
logger.warning(
"No SGYT found for the deliverable in the database !!!"
)
elif self.Deliverable.type in SEGY_3D:
if self.Deliverable.sgyt_master_status:
mnIL = get_item_through_dialogue(
self.parent, 'Minimum IL')
mxIL = get_item_through_dialogue(
self.parent, 'Maximum IL')
mnXL = get_item_through_dialogue(
self.parent, 'Minimum XL')
mxXL = get_item_through_dialogue(
self.parent, 'Maximum XL')
reel = get_item_through_dialogue(
self.parent, 'Tape number excluding prefix')
self.reel = str('{0:04d}'.format(int(reel)))
self.IL_range = [mnIL, mxIL]
self.XL_range = [mnXL, mxXL]
username = self.get_user_name()
if username == 'exit':
pass
else:
if len(username) != 0:
self.username = username
self.create_SEGY_3D_sgyt()
else:
self.get_user_name()
else:
logger.warning(
"No SGYT found for the deliverable in the database !!!"
)
elif caller == 'Sequence':
#self.sequence = self.disp_seq_dict[attribute]
self.sequence_list = [attribute]
# get min max IL and XL ranges
# mnIL = get_item_through_dialogue(self.parent,'Minimum IL')
# mxIL = get_item_through_dialogue(self.parent,'Maximum IL')
# mnXL = get_item_through_dialogue(self.parent, 'Minimum XL')
# mxXL = get_item_through_dialogue(self.parent, 'Maximum XL')
reel = get_item_through_dialogue(
self.parent, 'Tape number excluding prefix')
self.reel = str('{0:04d}'.format(int(reel)))
self.IL_range = [
0, 0
] # these need to be updated through PQ Man i the future
self.XL_range = [
0, 0
] # These need to be updated through PQ MAn in the future
username = self.get_user_name()
if username == 'exit':
pass
else:
if len(username) != 0:
self.username = username
self.create_sequence_wise_sgyt()
else:
self.get_user_name()
elif caller == 'Sequences':
#self.sequence = self.disp_seq_dict[attribute]
self.sequence_list = attribute
# get min max IL and XL ranges
# mnIL = get_item_through_dialogue(self.parent,'Minimum IL')
# mxIL = get_item_through_dialogue(self.parent,'Maximum IL')
# mnXL = get_item_through_dialogue(self.parent, 'Minimum XL')
# mxXL = get_item_through_dialogue(self.parent, 'Maximum XL')
reel = get_item_through_dialogue(
self.parent, 'Tape number excluding prefix')
self.reel = str('{0:04d}'.format(int(reel)))
self.IL_range = [
0, 0
] # these need to be updated through PQ Man i the future
self.XL_range = [
0, 0
] # These need to be updated through PQ MAn in the future
username = self.get_user_name()
if username == 'exit':
pass
else:
if len(username) != 0:
self.username = username
self.create_sequence_wise_sgyt()
else:
self.get_user_name()
elif ops == 'per_QC':
if caller == 'Deliverable':
self.Deliverable = self.disp_name_dict[attribute]
#Now check if the bin def and trc def file exists for the deliverable
if self.Deliverable.bin_def_status and self.Deliverable.trc_def_status:
# now the deliverable is set, time to set the dir service to the deliverable
self.dir_service.set_deliverable(self.Deliverable)
# return the list of SEGY files for the deliverable in the large files+ deliverable dir
self.choose_sgy_file_for_QC(ops)
else:
logger.warning(
"The bin.def and trc.def files for the deliberable are not defined in the database!!!"
)
if caller == 'file_sgy':
#fetch the bin def and trc def files from the database and create files in temp dir and SFTP them to the DUG WS
(bin_def_path, trc_def_path) = self.create_def_files_and_sFTP()
file_path = posixpath.join(
self.dir_service.data_dir_path_dict['data'], attribute)
logger.info("Now running Qc script on: " + file_path)
log_name = str(
self.Deliverable.id
) + "_" + self.Deliverable.name + "_" + attribute + '.headerlog'
log_dir = self.dir_service.data_dir_path_dict['headers']
log_path = posixpath.join(log_dir, log_name)
self.qc_log_path = log_path
# if log exists before delete it and than redo it
status = check_generic_path(self.DUG_connection_obj, log_path)
script_name = self.DUG_connection_obj.DUG_project_name + "_segy-hcm"
script_path = posixpath.join("/d/home/share/bin", script_name)
cmd = str(
"nohup " + script_path + " " + file_path + " -bd " +
bin_def_path + " -td " + trc_def_path + " -l " + log_path +
" > /dev/null 2>&1 &"
) # change this line to modify the command and the python package needs to be changed as well
if status == 'True':
message = "The log already exists either the QC is running or you are trying to run it again. Please check if the job is still running or if finished press ok to remove old file and create a new one !!"
self.pop_up_message_box = pop_up_message_box(
message, 'Warning')
self.pop_up_message_box.closed.connect(
lambda: self.run_segy_qc(cmd, True))
self.pop_up_message_box.show()
# modify the existing Db entry
db_entry = self.db_connection_obj.sess.query(
self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.
segy_on_disk_qc_log_path == log_path).first()
db_entry.segy_on_disk_qc_run_status = True
db_entry.segy_on_disk_qc_status = None
db_entry.segy_on_disk_qc_approver_name = None
db_entry.segy_on_disk_qc_approval_time = None
self.db_connection_obj.sess.commit()
else:
# execute the command on the DUG workstation
self.run_segy_qc(cmd, False)
db_entry = self.db_connection_obj.sess.query(
self.db_connection_obj.SEGY_QC_on_disk).filter(
self.db_connection_obj.SEGY_QC_on_disk.
segy_on_disk_file_path == file_path).first()
db_entry.segy_on_disk_qc_run_status = True
db_entry.segy_on_disk_qc_log_path = log_path
self.db_connection_obj.sess.commit()