def forward(self, job_num, final_filenames):
print("Start Time of READOUT IS: %s" % get_timestamp())
login_str = self._job_scratchpad.get_job_value(job_num, 'LOGIN_STR')
target_dir = self._job_scratchpad.get_job_value(job_num, 'TARGET_DIR')
results = {}
CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
ccds = list(final_filenames.keys())
for ccd in ccds:
final_file = final_filenames[ccd]
pathway = self._DAQ_PATH + final_file
with open(pathway) as file_to_calc:
if self.CHECKSUM_ENABLED:
data = file_to_calc.read()
resulting_md5 = hashlib.md5(data).hexdigest()
else:
resulting_md5 = '0'
minidict = {}
CCD_LIST.append(ccd)
CHECKSUM_LIST.append(resulting_md5)
FILENAME_LIST.append(target_dir + final_file)
cmd = 'scp ' + pathway + " " + login_str + target_dir + final_file
print("Finish Time of SCP'ing %s IS: %s" %
(pathway, get_timestamp()))
print("In forward() method, cmd is %s" % cmd)
os.system(cmd)
results['CCD_LIST'] = CCD_LIST
results['FILENAME_LIST'] = FILENAME_LIST
results['CHECKSUM_LIST'] = CHECKSUM_LIST
print("END Time of READOUT XFER IS: %s" % get_timestamp())
print("In forward method, results are: \n%s" % results)
return results
def forward(self, job_num, final_filenames):
print("Start Time of READOUT IS: %s" % get_timestamp())
login_str = self._job_scratchpad.get_job_value(job_num, 'LOGIN_STR')
target_dir = self._job_scratchpad.get_job_value(job_num, 'TARGET_DIR')
results = {}
CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
ccds = list(final_filenames.keys())
for ccd in ccds:
final_file = final_filenames[ccd]
pathway = self._DAQ_PATH + final_file
with open(pathway) as file_to_calc:
if self.CHECKSUM_ENABLED:
data = file_to_calc.read()
resulting_md5 = hashlib.md5(data).hexdigest()
else:
resulting_md5 = '0'
CCD_LIST.append(ccd)
CHECKSUM_LIST.append(resulting_md5)
FILENAME_LIST.append(target_dir + final_file)
cmd = 'scp ' + pathway + " " + login_str + target_dir + final_file
print("Finish Time of SCP'ing %s IS: %s" % (pathway, get_timestamp()))
print("In forward() method, cmd is %s" % cmd)
os.system(cmd)
results['CCD_LIST'] = CCD_LIST
results['FILENAME_LIST'] = FILENAME_LIST
results['CHECKSUM_LIST'] = CHECKSUM_LIST
print("END Time of READOUT XFER IS: %s" % get_timestamp())
print("In forward method, results are: \n%s" % results)
return results
def insufficient_base_resources(self, params, healthy_forwarders):
# send response msg to dmcs refusing job
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
ack_id = params['ACK_ID']
needed_workers = len(raft_list)
LOGGER.info('Reporting to DMCS that there are insufficient healthy forwarders for job #%s', job_num)
dmcs_params = {}
fail_dict = {}
dmcs_params[MSG_TYPE] = NEW_JOB_ACK
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[ACK_ID] = ack_id
### NOTE FOR DMCS ACK PROCESSING:
### if ACK_BOOL == True, there will NOT be a FAIL_DETAILS section
### If ACK_BOOL == False, there will always be a FAIL_DICT to examine AND there will always be a
### BASE_RESOURCES inside the FAIL_DICT
### If ACK_BOOL == False, and the BASE_RESOURCES inside FAIL_DETAILS == 0,
### there will be only NEEDED and AVAILABLE Forwarder params - nothing more
### If ACK_BOOL == False and BASE_RESOURCES inside FAIL_DETAILS == 1, there will always be a
### NCSA_RESOURCES inside FAIL_DETAILS set to either 0 or 'NO_RESPONSE'
### if NCSA_RESPONSE == 0, there will be NEEDED and AVAILABLE Distributor params
### if NCSA_RESOURCES == 'NO_RESPONSE' there will be nothing else
fail_dict['BASE_RESOURCES'] = '0'
fail_dict[NEEDED_FORWARDERS] = str(needed_workers)
fail_dict[AVAILABLE_FORWARDERS] = str(len(healthy_forwarders))
dmcs_params['FAIL_DETAILS'] = fail_dict
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
# mark job refused, and leave Forwarders in Idle state
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "JOB_ABORTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ABORTED_BASE_RESOURCES", get_timestamp())
idle_state = {"STATE": "IDLE"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_state)
return False
def ncsa_resources_query(self, params, healthy_forwarders):
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
LOGGER.info('Sufficient forwarders have been found. Checking NCSA')
self._pairs_dict = {}
forwarder_candidate_dict = {}
for i in range (0, needed_workers):
forwarder_candidate_dict[healthy_forwarders[i]] = raft_list[i]
self.FWD_SCBD.set_forwarder_status(healthy_forwarders[i], NCSA_RESOURCES_QUERY)
# Call this method for testing...
# There should be a message sent to NCSA here asking for available resources
timed_ack_id = self.get_next_timed_ack_id("NCSA_Ack")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY"
ncsa_params[JOB_NUM] = job_num
#ncsa_params[RAFT_NUM] = needed_workers
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = forwarder_candidate_dict
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "NCSA_RESOURCES_QUERY_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_NCSA_RESOURCES_QUERY_SENT", get_timestamp())
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info('The following forwarders have been sent to NCSA for pairing:')
LOGGER.info(forwarder_candidate_dict)
return timed_ack_id
def accept_job(self, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = NEW_JOB_ACK
dmcs_message[ACK_BOOL] = True
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED", get_timestamp())
self._base_publisher.publish_message("dmcs_consume", dmcs_message)
return True
def process_job_params(self, params):
transfer_params = params[TRANSFER_PARAMS]
self._job_scratchpad.set_job_transfer_params(params[JOB_NUM],
transfer_params)
self._job_scratchpad.set_job_value(job_number, "STATE",
"READY_WITH_PARAMS")
self._job_scratchpad.set_job_value(job_number,
"READY_WITH_PARAMS_TIME",
get_timestamp())
self.send_ack_response(DISTRIBUTOR_JOB_PARAMS_ACK, params)
def build_monitor_data(self, params):
monitor_data = {}
keez = list(params.keys())
for kee in keez:
monitor_data[kee] = params[kee]
monitor_data['SESSION_ID'] = self.get_current_session()
monitor_data['VISIT_ID'] = self.get_current_visit()
monitor_data['TIME'] = get_timestamp()
monitor_data['DATA_TYPE'] = self.DB_TYPE
return monitor_data
def accept_job(self, ack_id, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = self.PP_START_INTEGRATION_ACK
dmcs_message['COMPONENT'] = self.COMPONENT_NAME
dmcs_message[ACK_BOOL] = True
dmcs_message['ACK_ID'] = ack_id
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED",
get_timestamp())
self._base_publisher.publish_message("dmcs_ack_consume", dmcs_message)
return True
def forwarder_health_check(self, params):
job_num = str(params[JOB_NUM])
raft_list = params['RAFTS']
needed_workers = len(raft_list)
self.JOB_SCBD.add_job(job_num, needed_workers)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED", get_timestamp())
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED_E", get_epoch_timestamp())
LOGGER.info('Received new job %s. Needed workers is %s', job_num, needed_workers)
# run forwarder health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("FORWARDER_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_available_forwarders_list()
# Mark all healthy Forwarders Unknown
state_status = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.FWD_SCBD.set_forwarder_params(forwarders, state_status)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = FORWARDER_HEALTH_CHECK
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_RESOURCE_QUERY")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_RESOURCE_QUERY", get_timestamp())
audit_params = {}
audit_params['DATA_TYPE'] = 'FOREMAN_ACK_REQUEST'
audit_params['SUB_TYPE'] = 'FORWARDER_HEALTH_CHECK_ACK'
audit_params['ACK_ID'] = timed_ack
audit_parsms['COMPONENT_NAME'] = 'BASE_FOREMAN'
audit_params['TIME'] = get_epoch_timestamp()
for forwarder in forwarders:
self._base_publisher.publish_message(self.FWD_SCBD.get_value_for_forwarder(forwarder,"CONSUME_QUEUE"),
ack_params)
return timed_ack
def distribute_job_params(self, params, pairs):
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_PAIRS_ADDED", get_timestamp())
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwders = list(pairs.keys())
fwd_params = {}
fwd_params[MSG_TYPE] = "FORWARDER_JOB_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params[ACK_ID] = fwd_ack_id
for fwder in fwders:
fwd_params["TRANSFER_PARAMS"] = pairs[fwder]
route_key = self.FWD_SCBD.get_value_for_forwarder(fwder, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def process_health_check(self, params):
job_number = params[JOB_NUM]
self._job_scratchpad.set_job_value(job_number, "STATE", "ADD_JOB")
self._job_scratchpad.set_job_value(job_number, "ADD_JOB_TIME",
get_timestamp())
self.send_ack_response("DISTRIBUTOR_HEALTH_ACK", params)
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
date - get_timestamp()
self.JOB_SCBD.set_value_for_job(job_number, TIME_START_READOUT, date)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
distributors = [v['FQN'] for v in list(pairs.values())]
forwarders = list(pairs.keys())
ack_id = self.get_next_timed_ack_id('NCSA_READOUT')
### Send READOUT to NCSA with ACK_ID
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(NCSA_CONSUME, yaml.dump(ncsa_params))
self.ack_timer(4)
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
if ncsa_response['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('FORWARDER_READOUT')
for forwarder in forwarders:
name = self.FWD_SCBD.get_value_for_forwarder(forwarder, NAME)
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'FORWARDER_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder, START_READOUT)
self._publisher.publish_message(routing_key, yaml.dump(msg_params))
self.ack_timer(4)
forwarder_responses = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if len(forwarder_responses) == len(forwarders):
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = True
dmcs_params['COMMENT'] = "Readout begun at %s" % get_timestamp()
self._publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = 'Readout Failed: Problem at NCSA - Expected Distributor Acks is %s, Number of Distributor Acks received is %s' % (ncsa_response['EXPECTED_DISTRIBUTOR_ACKS'], ncsa_response['RECEIVED_DISTRIBUTOR_ACKS'])
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = "Readout Failed: No Response from NCSA"
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
def process_dmcs_new_job(self, params):
input_params = params
needed_workers = len(input_params[RAFTS])
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(7) # This is a HUGE num seconds for now..final setting will be milliseconds
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(timed_ack)
num_healthy_forwarders = len(healthy_forwarders)
if needed_workers > num_healthy_forwarders:
result = self.insufficient_base_resources(input_params, healthy_forwarders)
return result
else:
healthy_status = {"STATUS": "HEALTHY", "STATE":"READY_WITHOUT_PARAMS"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, healthy_status)
ack_id = self.ncsa_resources_query(input_params, healthy_forwarders)
self.ack_timer(3)
#Check ACK scoreboard for response from NCSA
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
pairs = {}
ack_bool = None
try:
ack_bool = ncsa_response[ACK_BOOL]
if ack_bool == True:
pairs = ncsa_response[PAIRS]
except KeyError as e:
pass
# Distribute job params and tell DMCS I'm ready.
if ack_bool == TRUE:
fwd_ack_id = self.distribute_job_params(input_params, pairs)
self.ack_timer(3)
fwd_params_response = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if fwd_params_response and (len(fwd_params_response) == len(fwders)):
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_TASK_PARAMS_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_TASK_PARAMS_SENT", get_timestamp())
in_ready_state = {'STATE':'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(fwders, in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(job_num)
else:
#not enough ncsa resources to do job - Notify DMCS
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_params)
result = self.insufficient_ncsa_resources(ncsa_response)
return result
else:
result = self.ncsa_no_response(input_params)
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(list(forwarder_candidate_dict.keys()), idle_params)
return result
def process_job_params(self, params):
transfer_params = params[TRANSFER_PARAMS]
self._job_scratchpad.set_job_transfer_params(params[JOB_NUM], transfer_params)
self._job_scratchpad.set_job_value(job_number, "STATE", "READY_WITH_PARAMS")
self._job_scratchpad.set_job_value(job_number, "READY_WITH_PARAMS_TIME", get_timestamp())
self.send_ack_response(DISTRIBUTOR_JOB_PARAMS_ACK, params)
def process_health_check(self, params):
job_number = params[JOB_NUM]
self._job_scratchpad.set_job_value(job_number, "STATE", "ADD_JOB")
self._job_scratchpad.set_job_value(job_number, "ADD_JOB_TIME", get_timestamp())
self.send_ack_response("DISTRIBUTOR_HEALTH_ACK", params)