def makedirs(self, filelist, endpoint): """ get list of dirs to make Parameters ---------- filelist : list The list of files endpoint : str The endpoint to use """ print "makedirs: filelist=", filelist dirlist = miscutils.get_list_directories(filelist) print "makedirs: dirlist=", dirlist for path in sorted( dirlist): # should already be sorted, but just in case miscutils.fwdebug(0, 'GLOBUS_ONLINE_DEBUG', 'endpoint=%s, path=%s' % (endpoint, path)) try: _ = self.goclient.endpoint_mkdir(endpoint, path) except Exception as e: if 'already exists' not in str(e): raise else: miscutils.fwdebug( 2, 'GLOBUS_ONLINE_DEBUG', 'already exists endpoint=%s, path=%s' % (endpoint, path))
def insert_dictionary_2Db(self, dbh, query, dictionary, debug=None): """Execute a query and return a cursor to a query :param query: string with query statement :param dictionary: dictionary to use in query :param debug: verbosity """ try: cur = dbh.cursor() cur.execute(query, dictionary) miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "dictionary into database " % (dictionary)) success = 1 #except cx_Oracle.IntegrityError as e: except cx_Oracle.DatabaseError as e: error, = e.args if error.code == 955: print('Table already exists') elif error.code == 1031: print("Insufficient privileges") print(error.code) print(error.message) print(error.context) success = 0 raise return success
def insert_dictionary_2Db(self, dbh, query, dictionary,debug=None): """Execute a query and return a cursor to a query :param query: string with query statement :param dictionary: dictionary to use in query :param debug: verbosity """ try: cur = dbh.cursor() cur.execute(query,dictionary) miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "dictionary into database " % (dictionary)) success = 1 #except cx_Oracle.IntegrityError as e: except cx_Oracle.DatabaseError as e: error, = e.args if error.code == 955: print('Table already exists') elif error.code == 1031: print("Insufficient privileges") print(error.code) print(error.message) print(error.context) success = 0 raise return success
def run_post_steps(filelist, config, fmobj): """ Performs steps necessary for each file """ #print config.keys() firstname = miscutils.parse_fullname(filelist[0], miscutils.CU_PARSE_FILENAME) filetype = dtsutils.determine_filetype(firstname) miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", "filetype = %s" % filetype) # dynamically load class specific to filetype classkey = 'dts_filetype_class_' + filetype filetype_class = miscutils.dynamically_load_class(config[classkey]) valdict = fmutils.get_config_vals({}, config, filetype_class.requested_config_vals()) ftobj = filetype_class(dbh=fmobj, config=valdict) for fullname in filelist: filename = miscutils.parse_fullname(fullname, miscutils.CU_PARSE_FILENAME) miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", "filename = %s" % filename) if dtsutils.check_already_registered(filename, fmobj): ftobj.post_steps(fullname) # e.g., Rasicam # if success fmobj.commit() else: print "File must already be registered in order to run post_steps"
def get_config_vals(archive_info, config, keylist): """ Search given dicts for specific values Parameters ---------- archive_info : dict Dictionary of the archive data config : dict Dictionary of the config data keylist : dict Dictionary of the keys to be searched for and whether they are required or optional Returns ------- dict The serach results """ info = {} for k, st in keylist.items(): if archive_info is not None and k in archive_info: info[k] = archive_info[k] elif config is not None and k in config: info[k] = config[k] elif st.lower() == 'req': miscutils.fwdebug(0, 'FMUTILS_DEBUG', '******************************') miscutils.fwdebug(0, 'FMUTILS_DEBUG', 'keylist = %s' % keylist) miscutils.fwdebug(0, 'FMUTILS_DEBUG', 'archive_info = %s' % archive_info) miscutils.fwdebug(0, 'FMUTILS_DEBUG', 'config = %s' % config) miscutils.fwdie('Error: Could not find required key (%s)' % k, 1, 2) return info
def start_transfer(self, filelist): """ activate src endpoint Parameters ---------- filelist : list List of files to transfer Returns ------- str of the globus task id """ src_endpoint = self.srcinfo['endpoint'] result = self.endpoint_activate(src_endpoint) # activate dst endpoint dst_endpoint = self.dstinfo['endpoint'] result = self.endpoint_activate(dst_endpoint) # create dst directories self.makedirs([finfo['dst'] for finfo in filelist.values()], dst_endpoint) ## Get a submission id: _, _, result = self.goclient.transfer_submission_id() self.submission_id = result["value"] miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\tsubmission id = %s" % self.submission_id) ## Create a transfer object: #t = Transfer(submission_id, src_endpoint, dst_endpoint, notify_on_succeeded = False, # notify_on_failed = False, notify_on_inactive= False, deadline='2m') deadline = datetime.utcnow() + timedelta(minutes=30) t = Transfer(self.submission_id, src_endpoint, dst_endpoint, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False, deadline=deadline) #print t.as_data() # add files to transfer for _, finfo in filelist.items(): sfile = finfo['src'] dfile = finfo['dst'] miscutils.fwdebug(2, 'GLOBUS_ONLINE_DEBUG', "\tadding to transfer %s = %s" % (sfile, dfile)) if sfile.endswith('/'): t.add_item(sfile, dfile, recursive=True) # error if true for file else: t.add_item(sfile, dfile) # start transfer _, _, result = self.goclient.transfer(t) task_id = result["task_id"] miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\ttask id = %s" % task_id) return task_id
def get_config_vals(archive_info, config, keylist): """ Search given dicts for specific values """ info = {} for k, st in keylist.items(): if archive_info is not None and k in archive_info: info[k] = archive_info[k] elif config is not None and k in config: info[k] = config[k] elif st.lower() == 'req': miscutils.fwdebug(0, 'FMUTILS_DEBUG', '******************************') miscutils.fwdebug(0, 'FMUTILS_DEBUG', f'keylist = {keylist}') miscutils.fwdebug(0, 'FMUTILS_DEBUG', f'archive_info = {archive_info}') miscutils.fwdebug(0, 'FMUTILS_DEBUG', f'config = {config}') miscutils.fwdie(f'Error: Could not find required key ({k})', 1, 2) return info
def get_archive_transfer_info(self): """ Return contents of ops_archive_transfer and ops_archive_transfer_val tables as a dictionary Returns ------- dict """ archive_transfer = collections.OrderedDict() sql = "select src,dst,transfer from ops_archive_transfer" curs = self.cursor() curs.execute(sql) for row in curs: if row[0] not in archive_transfer: archive_transfer[row[0]] = collections.OrderedDict() archive_transfer[row[0]][row[1]] = collections.OrderedDict( {'transfer': row[2]}) sql = "select src,dst,key,val from ops_archive_transfer_val" curs = self.cursor() curs.execute(sql) for row in curs: if row[0] not in archive_transfer: miscutils.fwdebug( 0, 'DESDBI_DEBUG', f"WARNING: found info in ops_archive_transfer_val for src archive {row[0]} which is not in ops_archive_transfer" ) archive_transfer[row[0]] = collections.OrderedDict() if row[1] not in archive_transfer[row[0]]: miscutils.fwdebug( 0, 'DESDBI_DEBUG', f"WARNING: found info in ops_archive_transfer_val for dst archive {row[1]} which is not in ops_archive_transfer" ) archive_transfer[row[0]][row[1]] = collections.OrderedDict() archive_transfer[row[0]][row[1]][row[2]] = row[3] return archive_transfer
def __del__(self): """ Do the semaphore signal and close DB connection """ if self.slot is not None and str(self.slot) != 'None': try: miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - signal") curs = self.dbh.cursor() curs.callproc("SEM_SIGNAL", [self.semname, self.slot]) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - signal") self.dbh.basic_update_row( 'SEMINFO', {'release_time': self.dbh.get_current_timestamp_str()}, {'id': self.id}) self.dbh.commit() except Exception as e: miscutils.fwdebug(0, "SEMAPHORE_DEBUG", "SEM - ERROR - " + str(e)) self.slot = None self.dbh.close()
def read_json_single(self, json_file, allMandatoryExposureKeys, debug): miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "reading file %s" % json_file) allExposures = [] my_header = {} numseq = {} all_exposures = dict() with open(json_file) as my_json: for line in my_json: all_data = json.loads(line) for key, value in all_data.items(): errorFlag = 0 if key == 'header': #read the values for the header (date and set_type are here) my_head = value allExposures.append(str(my_head['set_type'])) allExposures.append(str(my_head['createdAt'])) if key == 'exposures': #read all the exposures that were taken for the set_type in header my_header = value #Total Number of exposures in manifest file tot_exposures = len(my_header) if tot_exposures is None or tot_exposures == 0: raise Exception( "0 SN exposures parsed from json file") for i in range(tot_exposures): numseq = my_header[i]['sequence'] mytime = my_header[i]['acttime'] if mytime > 10 and numseq['seqnum'] == 2: first_expnum = my_header[i]['expid'] #Validate if acctime has a meaningful value. If acttime = 0.0, then it's a bad exposure. Skip it from the manifest. if mytime == 0.0: continue try: for mandatoryExposureKey in ( allMandatoryExposureKeys): miscutils.fwdebug( 3, 'DTSSNMANIFEST_DEBUG', "mandatory key %s" % mandatoryExposureKey) key = str(mandatoryExposureKey) if my_header[i][mandatoryExposureKey]: miscutils.fwdebug( 3, 'DTSSNMANIFEST_DEBUG', "mandatory key '%s' found %s" % (mandatoryExposureKey, my_header[i] [mandatoryExposureKey])) miscutils.fwdebug( 6, 'DTSSNMANIFEST_DEBUG', "allExposures in for: %s" % allExposures) try: if key == 'acttime': key = 'EXPTIME' all_exposures[key].append( my_header[i] [mandatoryExposureKey]) elif key == 'filter': key = 'BAND' all_exposures[key].append( str(my_header[i][ mandatoryExposureKey])) elif key == 'expid': key = 'EXPNUM' all_exposures[key].append( my_header[i] [mandatoryExposureKey]) else: all_exposures[key].append( my_header[i] [mandatoryExposureKey]) except KeyError: all_exposures[key] = [ my_header[i] [mandatoryExposureKey] ] except KeyError: miscutils.fwdebug( 0, 'DTSSNMANIFEST_DEBUG', "keyError: missing key %s in json entity: %s " % (mandatoryExposureKey, line)) errorFlag = 1 raise timestamp = all_exposures['date'][0] nite = dtsutils.convert_UTCstr_to_nite(timestamp) # get field by parsing set_type #print 'xxxx', my_head['set_type'] myfield = my_head['set_type'] if len(myfield) > 5: newfield = myfield[:5] else: newfield = myfield camsym = 'D' # no way to currently tell CAMSYM/INSTRUME from manifest file if not newfield.startswith('SN-'): raise ValueError( "Invalid field (%s). set_type = '%s'" % (newfield, my_head['set_type'])) #if json_file contains a path or compression extension, then cut it to only the filename jsonFile = miscutils.parse_fullname( json_file, miscutils.CU_PARSE_FILENAME) if tot_exposures is None or tot_exposures == 0: raise Exception( "0 SN exposures parsed from json file") for i in range(tot_exposures): if my_header[i]['acttime'] == 0.0: continue if i == 0: #all_exposures['FIELD'] = [str(my_head['set_type'])] all_exposures['FIELD'] = [newfield] all_exposures['CREATEDAT'] = [ str(my_head['createdAt']) ] all_exposures['MANIFEST_FILENAME'] = [jsonFile] all_exposures['NITE'] = [nite] all_exposures['SEQNUM'] = [1] all_exposures['CAMSYM'] = [camsym] else: #all_exposures['FIELD'].append(str(my_head['set_type'])) all_exposures['FIELD'].append(newfield) all_exposures['CREATEDAT'].append( str(my_head['createdAt'])) all_exposures['MANIFEST_FILENAME'].append( jsonFile) all_exposures['NITE'].append(nite) all_exposures['SEQNUM'].append(1) all_exposures['CAMSYM'].append(camsym) # Add the manifest filename value in the dictionary #all_exposures['MANIFEST_FILENAME'] = json_file miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "allExposures " % (all_exposures)) return all_exposures
def blocking_transfer(self, filelist): """ Do a blocking transfer Parameters ---------- filelist : list The files to transfer Returns ------- dict of the transfer results """ task_id = self.start_transfer(filelist) miscutils.fwdebug(0, 'GLOBUS_ONLINE_DEBUG', "\ttask_id = %s" % task_id) # wait for transfer to complete ## Check the progress of the new transfer: MAX_NUM_CHKS = 600 MAX_NUM_RETRY = 5 CHK_INTERVAL_SECS = 30 status = "ACTIVE" chk_cnt = 0 retry_cnt = 0 errstrs = {} while status == "ACTIVE" and chk_cnt < MAX_NUM_CHKS and retry_cnt < MAX_NUM_RETRY: miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "Checking transfer task status") status, reason, result = self.goclient.task(task_id) status = result["status"] miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\tstatus = %s" % result["status"]) miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\tfiles = %s" % result["files"]) miscutils.fwdebug( 1, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_total = %s" % result["subtasks_total"]) miscutils.fwdebug( 1, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_failed = %s" % result["subtasks_failed"]) miscutils.fwdebug( 1, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_retrying = %s" % result["subtasks_retrying"]) miscutils.fwdebug( 1, 'GLOBUS_ONLINE_DEBUG', "\tnice_status_details = %s" % result["nice_status_details"]) if status == "ACTIVE": chk_cnt += 1 # cannot call task_successful_transfers on task that is still active if result["nice_status_details"] is not None and result[ "nice_status_details"].startswith("Error"): # only print error message once if result["nice_status_details"] not in errstrs: print result["nice_status_details"] errstrs[result["nice_status_details"]] = True if result['subtasks_retrying'] != 0: retry_cnt += 1 else: miscutils.fwdebug(0, 'GLOBUS_ONLINE_DEBUG', "\tstatus = %s" % result["status"]) miscutils.fwdebug(0, 'GLOBUS_ONLINE_DEBUG', "\tfiles = %s" % result["files"]) miscutils.fwdebug( 0, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_total = %s" % result["subtasks_total"]) miscutils.fwdebug( 0, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_failed = %s" % result["subtasks_failed"]) miscutils.fwdebug( 0, 'GLOBUS_ONLINE_DEBUG', "\tsubtasks_retrying = %s" % result["subtasks_retrying"]) miscutils.fwdebug( 0, 'GLOBUS_ONLINE_DEBUG', "\tnice_status_details = %s" % result["nice_status_details"]) miscutils.fwdie("Error while transfering files", fmdefs.FM_EXIT_FAILURE) if chk_cnt < MAX_NUM_CHKS and retry_cnt < MAX_NUM_RETRY: time.sleep(CHK_INTERVAL_SECS) self.goclient.task_cancel(task_id) status, reason, successes = self.goclient.task_successful_transfers( task_id) print status print reason print "----------\n\n\n" print "subtask_list=", result print "\n\n\n" transresults = copy.deepcopy(filelist) if len(successes['DATA']) != len(filelist): for fname, _ in transresults.items(): transresults[fname]['err'] = 'problems transferring file' return transresults
def handle_bad_file(config, notify_file, delivery_fullname, dbh, filetype, metadata, disk_info, prov, msg): """ Perform steps required by any bad file """ dbh.rollback() # undo any db changes for this file miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "delivery_fullname = %s" % delivery_fullname) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "filetype = %s" % filetype) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "msg = %s" % msg) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "metadata = %s" % metadata) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "disk_info = %s" % disk_info) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "prov = %s" % prov) today = datetime.now() datepath = "%04d/%02d" % (today.year, today.month) # where is file now if disk_info is None: orig_fullname = delivery_fullname else: orig_fullname = disk_info['fullname'] # create a uniq name for living in the "bad file" area # contains relative path for storing in DB uniq_fullname = "%s/%s.%s" % (datepath, os.path.basename(orig_fullname), today.strftime("%Y%m%d%H%M%S%f")[:-3]) # absolute path destbad = "%s/%s" % (config['bad_file_dir'], uniq_fullname) if os.path.exists(destbad): miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "WARNING: bad file already exists (%s)" % destbad) os.remove(destbad) # make directory in "bad file" area and move file there miscutils.coremakedirs(os.path.dirname(destbad)) shutil.move(orig_fullname, destbad) # save information in db about bad file row = {} # save extra metadata if it exists if metadata is not None: badcols = dbh.get_column_names('DTS_BAD_FILE') for c in badcols: if c in metadata: row[c] = metadata[c] row['task_id'] = config['dts_task_id'] t = os.path.getmtime(notify_file) row['delivery_date'] = datetime.fromtimestamp(t) row['orig_filename'] = os.path.basename(orig_fullname) row['uniq_fullname'] = uniq_fullname row['rejected_date'] = today row['rejected_msg'] = msg row['filesize'] = os.path.getsize(destbad) if filetype is not None: row['filetype'] = filetype dbh.basic_insert_row('DTS_BAD_FILE', row) dbh.commit() os.unlink(notify_file)
def __init__(self, semname, task_id, desfile=None, section=None, connection=None, threaded=False): """ Create the DB connection and do the semaphore wait. """ self.desfile = desfile self.section = section self.semname = semname self.task_id = task_id self.slot = None miscutils.fwdebug(3, f"SEMAPHORE_DEBUG", "SEM - INFO - semname {self.semname}") miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - db-specific imports") import despydmdb.desdmdbi as desdmdbi import cx_Oracle miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - db-specific imports") miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - db connection") self.dbh = desdmdbi.DesDmDbi(desfile, section, threaded=threaded) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - db connection") curs = self.dbh.cursor() sql = f"select count(*) from semlock where name={self.dbh.get_named_bind_string('name')}" curs.execute(sql, {'name': semname}) num_slots = curs.fetchone()[0] if num_slots == 0: miscutils.fwdebug(0, "SEMAPHORE_DEBUG", f"SEM - ERROR - no locks with name {semname}") raise ValueError(f'No locks with name {semname}') self.id = self.dbh.get_seq_next_value('seminfo_seq') self.dbh.basic_insert_row( 'seminfo', { 'id': self.id, 'name': self.semname, 'request_time': self.dbh.get_current_timestamp_str(), 'task_id': task_id, 'num_slots': num_slots }) self.dbh.commit() self.slot = curs.var(cx_Oracle.NUMBER) done = False trycnt = 1 while not done and trycnt <= MAXTRIES: try: miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - wait") curs.callproc("SEM_WAIT", [self.semname, self.slot]) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - wait") miscutils.fwdebug(3, "SEMAPHORE_DEBUG", f"SEM - INFO - slot {self.slot}") done = True if not self.dbh.is_oracle(): self.dbh.commit() # test database must commit except Exception as e: miscutils.fwdebug(0, "SEMAPHORE_DEBUG", f"SEM - ERROR - {str(e)}") time.sleep(TRYINTERVAL) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - remake db connection") self.dbh = desdmdbi.DesDmDbi(desfile, section) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - remake db connection") curs = self.dbh.cursor() self.slot = curs.var(cx_Oracle.NUMBER) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - BEG - dequeue") curs.callproc("SEM_DEQUEUE", [self.semname, self.slot]) miscutils.fwdebug(3, "SEMAPHORE_DEBUG", "SEM - END - dequeue") trycnt += 1 if done: # need different connection to do the commit of the grant info as commit will release lock dbh2 = desdmdbi.DesDmDbi(desfile, section, connection) dbh2.basic_update_row( 'SEMINFO', { 'grant_time': dbh2.get_current_timestamp_str(), 'num_requests': trycnt, 'slot': self.slot }, {'id': self.id}) dbh2.commit()
def ingestAllExposures(self, allExposures,dbh, debug=None): """ Ingest all the exposures in EXPOSURES_IN_MANIFEST and SN_SUBMIT_REQUEST #If SEQNUM is > 1, then it means the same field was taken again during the same night. #This will only happens in rare occasion when the sequence had to be aborted before it finished. :param allExposures: Dictionary with the following keys: [set_type,createdAt,expid,object,date,acttime,filter] """ newdictionary = {} for key in ['CAMSYM', 'EXPNUM','MANIFEST_FILENAME','FIELD','BAND','EXPTIME', 'NITE']: newdictionary[key] = allExposures[key] #print "xx",allExposures dict2Ingest = {} for i in range(len(allExposures['EXPTIME'])): for key in newdictionary.keys(): keytoingest = key valuetoingest = newdictionary[key][i] dict2Ingest[keytoingest] = valuetoingest miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "dict2Ingest %s " % (dict2Ingest)) try: sqlInsertExposuresInManifest = """insert into MANIFEST_EXPOSURE (CAMSYM,EXPNUM,MANIFEST_FILENAME,NITE,FIELD,BAND,EXPTIME) VALUES (:CAMSYM, :EXPNUM, :MANIFEST_FILENAME, :NITE, :FIELD, :BAND, :EXPTIME)""" miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "sqlInsertExposuresInManifest %s " % (sqlInsertExposuresInManifest)) success = self.insert_dictionary_2Db(dbh, sqlInsertExposuresInManifest, dict2Ingest, debug=debug) if success: miscutils.fwdebug(1, 'DTSSNMANIFEST_DEBUG', "Insert into EXPOSURES_IN_MANIFEST was successful..") except cx_Oracle.IntegrityError as e: print "error while inserting into EXPOSURES_IN_MANIFEST: ", e raise ######################################################################################## # #Fix first expnum. First expnum is the first exposure for each filter set. In case of #one a field with one filter exposure, then first_expnum = expnum. #For more than one exposure / band/field, then first_expnum = first exposure of set. # #Determine index of list for exptime = 10. (poiting exposure) allexps= allExposures['EXPTIME'] miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "all exptimes %s" % (allexps)) for i,val in enumerate(allexps): if val == 10.0: pointingIndex = i miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "pointing Exposure index is %s" % pointingIndex) #find where there are repetead bands, but exclude the band where the exptime = 10 ListofBands = allExposures['BAND'] miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "listOfaBands...%s" % ListofBands) bandsDicIndexes = defaultdict(list) for i,item in enumerate(allExposures['BAND']): bandsDicIndexes[item].append(i) #I have to loop trorugh the dictionary for all the bands. Cound how many bands. Get the vaues from this dictionary #which is the index to the list, and use that to determine the elementes for all the other dictionaries. #I need the follwoing elementsl 'FIELD','NITE','BAND','MANIFEST_FILENAME','FIRST_EXPNUM','SEQNUM' ind2use = [] flag_first = 0 for ind, b in enumerate(ListofBands): miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "indexes %s %s" % (bandsDicIndexes[b], ind)) if ind == pointingIndex: miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "found pointing index %s %s " % (ind, pointingIndex)) continue else: #for two exposures and one of them is the poiting if len(bandsDicIndexes[b]) <= 2 and ind == pointingIndex+1: ind2use.append((max(bandsDicIndexes[b]))) #print "the index", ind2use #if there are more than 2 exposures (generally for deep fields elif len(bandsDicIndexes[b]) > 2 and ind == pointingIndex+1: ind2use.append(bandsDicIndexes[b][ind]) flag_first = 1 elif len(bandsDicIndexes[b]) == 1: ind2use.append(bandsDicIndexes[b][0]) elif len(bandsDicIndexes[b]) == 2 and ind != pointingIndex and flag_first==0: ind2use.append(min(bandsDicIndexes[b])) flag_first = 1 if flag_first: break #contruct the dictionary with only the elements that needs to go into the DB #To do this use the ind2use extracted from the above list. newDic = {} for index in ind2use: #print index newDic['FIELD'] = allExposures['FIELD'][index] newDic['NITE'] = allExposures['NITE'][index] newDic['BAND'] = allExposures['BAND'][index] newDic['MANIFEST_FILENAME'] = allExposures['MANIFEST_FILENAME'][index] newDic['FIRST_EXPNUM'] = allExposures['EXPNUM'][index] newDic['SEQNUM'] = allExposures['SEQNUM'][index] miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "index=%s, newDic=%s" % (index, newDic)) #Ingest into the database each of them try: sqlInsertSNSubmitRequest = """insert into SN_SUBMIT_REQUEST (FIELD,NITE,BAND,MANIFEST_FILENAME,FIRST_EXPNUM,SEQNUM) VALUES (:FIELD, :NITE, :BAND, :MANIFEST_FILENAME, :FIRST_EXPNUM, :SEQNUM)""" miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "sqlInsertSNSubmitRequest = %s" % sqlInsertExposuresInManifest) success = self.insert_dictionary_2Db(dbh,sqlInsertSNSubmitRequest, newDic,debug=debug) if success: miscutils.fwdebug(1, 'DTSSNMANIFEST_DEBUG', "Insert into SN_SUBMIT_REQUEST was successful..") except cx_Oracle.IntegrityError as e: print "error while inserting into SN_SUBMIT_REQUEST: ", e raise
def load_artifact_gtt(self, filelist): """ insert file artifact information into global temp table Parameters ---------- filelist : list List of dictionaries, one for each file, giving the file metadata to store. Returns ------- str The name of the temp table """ # filelist is list of file dictionaries # returns artifact GTT table name parsemask = miscutils.CU_PARSE_FILENAME | miscutils.CU_PARSE_EXTENSION # make sure table is empty before loading it self.empty_gtt(dmdbdefs.DB_GTT_ARTIFACT) colmap = [ dmdbdefs.DB_COL_FILENAME, dmdbdefs.DB_COL_COMPRESSION, dmdbdefs.DB_COL_MD5SUM, dmdbdefs.DB_COL_FILESIZE ] rows = [] for _file in filelist: miscutils.fwdebug(3, 'DESDBI_DEBUG', f"file = {_file}") fname = None comp = None md5sum = None filesize = None if dmdbdefs.DB_COL_FILENAME in _file or dmdbdefs.DB_COL_FILENAME.lower( ) in _file: if dmdbdefs.DB_COL_COMPRESSION in _file: fname = _file[dmdbdefs.DB_COL_FILENAME] comp = _file[dmdbdefs.DB_COL_COMPRESSION] elif dmdbdefs.DB_COL_COMPRESSION.lower() in _file: fname = _file[dmdbdefs.DB_COL_FILENAME.lower()] comp = _file[dmdbdefs.DB_COL_COMPRESSION.lower()] elif dmdbdefs.DB_COL_FILENAME in _file: (fname, comp) = miscutils.parse_fullname( _file[dmdbdefs.DB_COL_FILENAME], parsemask) else: (fname, comp) = miscutils.parse_fullname( _file[dmdbdefs.DB_COL_FILENAME.lower()], parsemask) miscutils.fwdebug(3, 'DESDBI_DEBUG', f"fname={fname}, comp={comp}") elif 'fullname' in _file: (fname, comp) = miscutils.parse_fullname(_file['fullname'], parsemask) miscutils.fwdebug( 3, 'DESDBI_DEBUG', f"parse_fullname: fname={fname}, comp={comp}") else: miscutils.fwdebug(3, 'DESDBI_DEBUG', f"file={_file}") raise ValueError(f"Invalid entry filelist({_file})") if dmdbdefs.DB_COL_FILESIZE in _file: filesize = _file[dmdbdefs.DB_COL_FILESIZE] elif dmdbdefs.DB_COL_FILESIZE.lower() in _file: filesize = _file[dmdbdefs.DB_COL_FILESIZE.lower()] if dmdbdefs.DB_COL_MD5SUM in _file: md5sum = _file[dmdbdefs.DB_COL_MD5SUM] elif dmdbdefs.DB_COL_MD5SUM.lower() in _file: md5sum = _file[dmdbdefs.DB_COL_MD5SUM.lower()] miscutils.fwdebug( 3, 'DESDBI_DEBUG', f"row: fname={fname}, comp={comp}, filesize={filesize}, md5sum={md5sum}" ) rows.append({ dmdbdefs.DB_COL_FILENAME: fname, dmdbdefs.DB_COL_COMPRESSION: comp, dmdbdefs.DB_COL_FILESIZE: filesize, dmdbdefs.DB_COL_MD5SUM: md5sum }) self.insert_many(dmdbdefs.DB_GTT_ARTIFACT, colmap, rows) return dmdbdefs.DB_GTT_ARTIFACT
def handle_file(notify_file, delivery_fullname, config, filemgmt, task_id): """ Performs steps necessary for each file """ filetype = None metadata = None disk_info = None prov = None # read values from notify file notifydict = read_notify_file(notify_file) # use dts_md5sum from notify_file dts_md5sum = None if 'md5sum' in notifydict: dts_md5sum = notifydict['md5sum'] print "%s: dts md5sum = %s" % (delivery_fullname, dts_md5sum) #print config.keys() try: filename = miscutils.parse_fullname(delivery_fullname, miscutils.CU_PARSE_FILENAME) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "filename = %s" % filename) if not os.path.exists(delivery_fullname): print "Warning: delivered file does not exist:" print "\tnotification file: %s" % notify_file print "\tdelivered file: %s" % delivery_fullname print "\tRemoving notification file and continuing" os.unlink(notify_file) return if dts_md5sum is not None: starttime = datetime.now() fileinfo_before_move = diskutils.get_single_file_disk_info(deliver_fullname, True, None) endtime = datetime.now() print "%s: md5sum before move %s (%0.2f secs)" % (delivery_fullname, fileinfo_before_move['md5sum'], endtime-starttime) if fileinfo_before_move['md5sum'] != dts_md5sum: print "%s: dts md5sum = %s" % (delivery_fullname, dts_md5sum) print "%s: py md5sum = %s" % (delivery_fullname, fileinfo_before_move['md5sum']) raise Exception("Error: md5sum in delivery dir not the same as DTS-provided md5sum") if not dtsutils.check_already_registered(filename, filemgmt): filetype = dtsutils.determine_filetype(filename) miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", "filetype = %s" % filetype) # dynamically load class specific to filetype classkey = 'dts_filetype_class_' + filetype filetype_class = miscutils.dynamically_load_class(config[classkey]) valDict = fmutils.get_config_vals({}, config, filetype_class.requested_config_vals()) filetypeObj = filetype_class(dbh=filemgmt, config=valDict) metadata = filetypeObj.get_metadata(delivery_fullname) metadata['filename'] = filename metadata['filetype'] = filetype miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", 'len(metadata) = %s' % len(metadata)) miscutils.fwdebug(6, "DTSFILEHANDLER_DEBUG", 'metadata = %s' % metadata) filetypeObj.check_valid(delivery_fullname) # should raise exception if not valid prov = generate_provenance(delivery_fullname) miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", 'archive_rel_path = %s' % archive_rel_path) miscutils.fwdebug(3, "DTSFILEHANDLER_DEBUG", 'prov = %s' % prov) archive_rel_path = filetypeObj.get_archive_path(delivery_fullname) disk_info = move_file_to_archive(config, delivery_fullname, archive_rel_path, dts_md5sum) save_data_db(filemgmt, task_id, {'file_1': metadata}, disk_info, prov) filetypeObj.post_steps(disk_info['fullname']) # e.g., Rasicam # if success filemgmt.commit() os.unlink(notify_file) else: handle_bad_file(config, notify_file, delivery_fullname, filemgmt, filetype, metadata, disk_info, prov, "already registered") except Exception as err: (type, value, trback) = sys.exc_info() print "******************************" print "Error: %s" % delivery_fullname traceback.print_exception(type, value, trback, file=sys.stdout) print "******************************" handle_bad_file(config, notify_file, delivery_fullname, filemgmt, filetype, metadata, disk_info, prov, "Exception: %s" % err) except SystemExit: # Wrappers code calls exit if cannot find header value handle_bad_file(config, notify_file, delivery_fullname, filemgmt, filetype, metadata, disk_info, prov, "SystemExit: Probably missing header value. Check log for error msg.") filemgmt.commit()
def ingestAllExposures(self, allExposures, dbh, debug=None): """ Ingest all the exposures in EXPOSURES_IN_MANIFEST and SN_SUBMIT_REQUEST #If SEQNUM is > 1, then it means the same field was taken again during the same night. #This will only happens in rare occasion when the sequence had to be aborted before it finished. :param allExposures: Dictionary with the following keys: [set_type,createdAt,expid,object,date,acttime,filter] """ newdictionary = {} for key in [ 'CAMSYM', 'EXPNUM', 'MANIFEST_FILENAME', 'FIELD', 'BAND', 'EXPTIME', 'NITE' ]: newdictionary[key] = allExposures[key] #print "xx",allExposures dict2Ingest = {} for i in range(len(allExposures['EXPTIME'])): for key in newdictionary.keys(): keytoingest = key valuetoingest = newdictionary[key][i] dict2Ingest[keytoingest] = valuetoingest miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "dict2Ingest %s " % (dict2Ingest)) try: sqlInsertExposuresInManifest = """insert into MANIFEST_EXPOSURE (CAMSYM,EXPNUM,MANIFEST_FILENAME,NITE,FIELD,BAND,EXPTIME) VALUES (:CAMSYM, :EXPNUM, :MANIFEST_FILENAME, :NITE, :FIELD, :BAND, :EXPTIME)""" miscutils.fwdebug( 3, 'DTSSNMANIFEST_DEBUG', "sqlInsertExposuresInManifest %s " % (sqlInsertExposuresInManifest)) success = self.insert_dictionary_2Db( dbh, sqlInsertExposuresInManifest, dict2Ingest, debug=debug) if success: miscutils.fwdebug( 1, 'DTSSNMANIFEST_DEBUG', "Insert into EXPOSURES_IN_MANIFEST was successful..") except cx_Oracle.IntegrityError as e: print "error while inserting into EXPOSURES_IN_MANIFEST: ", e raise ######################################################################################## # #Fix first expnum. First expnum is the first exposure for each filter set. In case of #one a field with one filter exposure, then first_expnum = expnum. #For more than one exposure / band/field, then first_expnum = first exposure of set. # #Determine index of list for exptime = 10. (poiting exposure) allexps = allExposures['EXPTIME'] miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "all exptimes %s" % (allexps)) for i, val in enumerate(allexps): if val == 10.0: pointingIndex = i miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "pointing Exposure index is %s" % pointingIndex) #find where there are repetead bands, but exclude the band where the exptime = 10 ListofBands = allExposures['BAND'] miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "listOfaBands...%s" % ListofBands) bandsDicIndexes = defaultdict(list) for i, item in enumerate(allExposures['BAND']): bandsDicIndexes[item].append(i) #I have to loop trorugh the dictionary for all the bands. Cound how many bands. Get the vaues from this dictionary #which is the index to the list, and use that to determine the elementes for all the other dictionaries. #I need the follwoing elementsl 'FIELD','NITE','BAND','MANIFEST_FILENAME','FIRST_EXPNUM','SEQNUM' ind2use = [] flag_first = 0 for ind, b in enumerate(ListofBands): miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "indexes %s %s" % (bandsDicIndexes[b], ind)) if ind == pointingIndex: miscutils.fwdebug( 3, 'DTSSNMANIFEST_DEBUG', "found pointing index %s %s " % (ind, pointingIndex)) continue else: #for two exposures and one of them is the poiting if len(bandsDicIndexes[b]) <= 2 and ind == pointingIndex + 1: ind2use.append((max(bandsDicIndexes[b]))) #print "the index", ind2use #if there are more than 2 exposures (generally for deep fields elif len(bandsDicIndexes[b]) > 2 and ind == pointingIndex + 1: ind2use.append(bandsDicIndexes[b][ind]) flag_first = 1 elif len(bandsDicIndexes[b]) == 1: ind2use.append(bandsDicIndexes[b][0]) elif len(bandsDicIndexes[b] ) == 2 and ind != pointingIndex and flag_first == 0: ind2use.append(min(bandsDicIndexes[b])) flag_first = 1 if flag_first: break #contruct the dictionary with only the elements that needs to go into the DB #To do this use the ind2use extracted from the above list. newDic = {} for index in ind2use: #print index newDic['FIELD'] = allExposures['FIELD'][index] newDic['NITE'] = allExposures['NITE'][index] newDic['BAND'] = allExposures['BAND'][index] newDic['MANIFEST_FILENAME'] = allExposures['MANIFEST_FILENAME'][ index] newDic['FIRST_EXPNUM'] = allExposures['EXPNUM'][index] newDic['SEQNUM'] = allExposures['SEQNUM'][index] miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "index=%s, newDic=%s" % (index, newDic)) #Ingest into the database each of them try: sqlInsertSNSubmitRequest = """insert into SN_SUBMIT_REQUEST (FIELD,NITE,BAND,MANIFEST_FILENAME,FIRST_EXPNUM,SEQNUM) VALUES (:FIELD, :NITE, :BAND, :MANIFEST_FILENAME, :FIRST_EXPNUM, :SEQNUM)""" miscutils.fwdebug( 3, 'DTSSNMANIFEST_DEBUG', "sqlInsertSNSubmitRequest = %s" % sqlInsertExposuresInManifest) success = self.insert_dictionary_2Db(dbh, sqlInsertSNSubmitRequest, newDic, debug=debug) if success: miscutils.fwdebug( 1, 'DTSSNMANIFEST_DEBUG', "Insert into SN_SUBMIT_REQUEST was successful..") except cx_Oracle.IntegrityError as e: print "error while inserting into SN_SUBMIT_REQUEST: ", e raise
def read_json_single(self, json_file,allMandatoryExposureKeys, debug): miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "reading file %s" % json_file) allExposures = [] my_header = {} numseq = {} all_exposures = dict() with open(json_file) as my_json: for line in my_json: all_data = json.loads(line) for key, value in all_data.items(): errorFlag = 0 if key == 'header': #read the values for the header (date and set_type are here) my_head = value allExposures.append(str(my_head['set_type'])) allExposures.append(str(my_head['createdAt'])) if key == 'exposures': #read all the exposures that were taken for the set_type in header my_header = value #Total Number of exposures in manifest file tot_exposures = len(my_header) if tot_exposures is None or tot_exposures == 0: raise Exception("0 SN exposures parsed from json file") for i in range(tot_exposures): numseq = my_header[i]['sequence'] mytime = my_header[i]['acttime'] if mytime > 10 and numseq['seqnum'] == 2: first_expnum = my_header[i]['expid'] #Validate if acctime has a meaningful value. If acttime = 0.0, then it's a bad exposure. Skip it from the manifest. if mytime == 0.0: continue try: for mandatoryExposureKey in (allMandatoryExposureKeys): miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "mandatory key %s" % mandatoryExposureKey) key = str(mandatoryExposureKey) if my_header[i][mandatoryExposureKey]: miscutils.fwdebug(3, 'DTSSNMANIFEST_DEBUG', "mandatory key '%s' found %s" % (mandatoryExposureKey, my_header[i][mandatoryExposureKey])) miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "allExposures in for: %s" % allExposures) try: if key == 'acttime': key = 'EXPTIME' all_exposures[key].append(my_header[i][mandatoryExposureKey]) elif key == 'filter': key = 'BAND' all_exposures[key].append(str(my_header[i][mandatoryExposureKey])) elif key == 'expid': key = 'EXPNUM' all_exposures[key].append(my_header[i][mandatoryExposureKey]) else: all_exposures[key].append(my_header[i][mandatoryExposureKey]) except KeyError: all_exposures[key] = [my_header[i][mandatoryExposureKey]] except KeyError: miscutils.fwdebug(0, 'DTSSNMANIFEST_DEBUG', "keyError: missing key %s in json entity: %s " % (mandatoryExposureKey,line)) errorFlag = 1 raise timestamp = all_exposures['date'][0] nite = dtsutils.convert_UTCstr_to_nite(timestamp) # get field by parsing set_type #print 'xxxx', my_head['set_type'] myfield = my_head['set_type'] if len(myfield) > 5: newfield = myfield[:5] else: newfield = myfield camsym = 'D' # no way to currently tell CAMSYM/INSTRUME from manifest file if not newfield.startswith('SN-'): raise ValueError("Invalid field (%s). set_type = '%s'" % (newfield, my_head['set_type'])) #if json_file contains a path or compression extension, then cut it to only the filename jsonFile = miscutils.parse_fullname(json_file, miscutils.CU_PARSE_FILENAME) if tot_exposures is None or tot_exposures == 0: raise Exception("0 SN exposures parsed from json file") for i in range(tot_exposures): if my_header[i]['acttime'] == 0.0: continue if i == 0: #all_exposures['FIELD'] = [str(my_head['set_type'])] all_exposures['FIELD'] = [newfield] all_exposures['CREATEDAT'] = [str(my_head['createdAt'])] all_exposures['MANIFEST_FILENAME'] = [jsonFile] all_exposures['NITE'] = [nite] all_exposures['SEQNUM'] = [1] all_exposures['CAMSYM'] = [camsym] else: #all_exposures['FIELD'].append(str(my_head['set_type'])) all_exposures['FIELD'].append(newfield) all_exposures['CREATEDAT'].append(str(my_head['createdAt'])) all_exposures['MANIFEST_FILENAME'].append(jsonFile) all_exposures['NITE'].append(nite) all_exposures['SEQNUM'].append(1) all_exposures['CAMSYM'].append(camsym) # Add the manifest filename value in the dictionary #all_exposures['MANIFEST_FILENAME'] = json_file miscutils.fwdebug(6, 'DTSSNMANIFEST_DEBUG', "allExposures " % (all_exposures)) return all_exposures