def query(self, arg): """ Execute an SQL query on the database and return the result as LIGO_LW XML @param arg: a text string containing an SQL query to be executed @return: None """ global logger global xmlparser, dbobj # get the query string and log it querystr = arg[0] logger.debug("Method query called with %s" % querystr) # assume failure code = 1 try: # create a ligo metadata object lwtparser = ldbd.LIGOLwParser() ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj) # execute the query rowcount = ligomd.select(querystr) # convert the result to xml result = ligomd.xml() logger.debug("Method query: %d rows returned" % rowcount) code = 0 except Exception, e: result = ("Error querying metadata database: %s" % e) logger.error(result)
def __init__(self, client): def dtd_uri_callback(uri): if uri == 'http://ldas-sw.ligo.caltech.edu/doc/ligolwAPI/html/ligolw_dtd.txt': return 'file://localhost' + os.path.join( os.environ["GLUE_PREFIX"], 'etc/ligolw_dtd.txt' ) else: return uri self.client = client self.xmlparser = pyRXP.Parser() self.xmlparser.eoCB = dtd_uri_callback self.lwtparser = ldbd.LIGOLwParser() self.ligomd = ldbd.LIGOMetadata(self.xmlparser, self.lwtparser, None)
def setupSegment_md(filename, xmlparser, lwtparser, debug): """ Helper function used to setup ligolw parser (S6 xml generation tool). """ segment_md = ldbd.LIGOMetadata(xmlparser, lwtparser) #if debug: # print "Inserting file %s." % filename fh = open(filename, 'r') xmltext = fh.read() fh.close() segment_md.parse(xmltext) if debug: #segment_md.table segment_md.table.keys() return segment_md
# pick off the format input type format = environ['wsgiorg.routing_args'][1]['format'] # for a POST the format must be JSON if format != 'json': start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request\n\nformat must be 'json' for POST operation" return [msg] logger.debug("Method query called with '%s'" % querystr) try: # create a ligo metadata object lwtparser = ldbd.LIGOLwParser() ligomd = ldbd.LIGOMetadata(xmlparser, lwtparser, dbobj) # execute the query rowcount = ligomd.select(querystr) # convert the result to xml result = ligomd.xml() logger.debug("Method query: %d rows returned" % rowcount) except Exception, e: start_response("500 Internal Server Error", [('Content-type', 'text/plain')]) msg = "500 Internal Server Error\n\n%s" % e logger.error(msg) return [msg]
def insertdmt(self, environ, start_response): """ """ global xmlparser, dbobj global dmt_proc_dict, dmt_seg_def_dict, creator_db proc_key = {} known_proc = {} seg_def_key = {} logger.debug( "Method dmtinsert called." ) logger.debug( "Known processes %s, " % str(dmt_proc_dict) ) logger.debug( "Known segment_definers %s" % str(dmt_seg_def_dict) ) # use specific grid-mapfile for insertdmt operation mapfile = self.configuration['gridmap_insertdmt'] # check authorization authorized, subject = self.checkAuthorizationGridMap(environ, mapfile) if not authorized: start_response("401 Unauthorized", [('Content-type', 'text/plain')]) msg = "401 Unauthorized\n\nSubject %s is not authorized for method insertdmt" % subject logger.info("Subject %s is not authorized for method insertdmt" % subject) return [ msg ] else: logger.info("Subject %s is authorized for method insertdmt" % subject) # pick off the format input type format = environ['wsgiorg.routing_args'][1]['format'] # for a POST the format must be JSON if format != 'json': start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request\n\nformat must be 'json' for POST operation" return [ msg ] # read the incoming payload try: inputString = cjson.decode(environ['wsgi.input'].read()) except Exception as e: start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request" logger.debug("Error decoding input: %s" % e) return [ msg ] logger.debug("Method insertdmt called with '%s'" % inputString) try: # create a ligo metadata object lwtparser = ldbd.LIGOLwParser() ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj) # parse the input string into a metadata object logger.debug("parsing xml data") ligomd.parse(inputString) # store the users dn in the process table ligomd.set_dn(subject) # determine the local creator_db number if creator_db is None: sql = "SELECT DEFAULT FROM SYSCAT.COLUMNS WHERE " sql += "TABNAME = 'PROCESS' AND COLNAME = 'CREATOR_DB'" ligomd.curs.execute(sql) creator_db = ligomd.curs.fetchone()[0] # determine the locations of columns we need in the process table process_cols = ligomd.table['process']['orderedcol'] node_col = process_cols.index('node') prog_col = process_cols.index('program') upid_col = process_cols.index('unix_procid') start_col = process_cols.index('start_time') end_col = process_cols.index('end_time') pid_col = process_cols.index('process_id') # determine and remove known entries from the process table rmv_idx = [] for row_idx,row in enumerate(ligomd.table['process']['stream']): uniq_proc = (row[node_col],row[prog_col],row[upid_col],row[start_col]) try: proc_key[str(row[pid_col])] = dmt_proc_dict[uniq_proc] known_proc[str(dmt_proc_dict[uniq_proc])] = row[end_col] rmv_idx.append(row_idx) except KeyError: # we know nothing about this process, so query the database sql = "SELECT BLOB(process_id) FROM process WHERE " sql += "creator_db = " + str(creator_db) + " AND " sql += "node = '" + row[node_col] + "' AND " sql += "program = '" + row[prog_col] + "' AND " sql += "unix_procid = " + str(row[upid_col]) + " AND " sql += "start_time = " + str(row[start_col]) ligomd.curs.execute(sql) db_proc_ids = ligomd.curs.fetchall() if len(db_proc_ids) == 0: # this is a new process with no existing entry dmt_proc_dict[uniq_proc] = row[pid_col] elif len(db_proc_ids) == 1: # the process_id exists in the database so use that insted dmt_proc_dict[uniq_proc] = db_proc_ids[0][0] proc_key[str(row[pid_col])] = dmt_proc_dict[uniq_proc] known_proc[str(dmt_proc_dict[uniq_proc])] = row[end_col] rmv_idx.append(row_idx) else: # multiple entries for this process, needs human assistance raise ServerHandlerException("multiple entries for dmt process") # delete the duplicate processs rows and clear the table if necessary newstream = [] for row_idx,row in enumerate(ligomd.table['process']['stream']): try: rmv_idx.index(row_idx) except ValueError: newstream.append(row) ligomd.table['process']['stream'] = newstream if len(ligomd.table['process']['stream']) == 0: del ligomd.table['process'] # delete the duplicate process_params rows and clear the table if necessary # (the DMT does not write a process_params table, so check for one first) if 'process_params' in ligomd.table: ppid_col = ligomd.table['process_params']['orderedcol'].index('process_id') newstream = [] for row_idx,row in enumerate(ligomd.table['process_params']['stream']): # if the process_id in this row is known, delete (i.e. don't copy) it try: proc_key[str(row[ppid_col])] except KeyError: newstream.append(row) ligomd.table['process_params']['stream'] = newstream if len(ligomd.table['process_params']['stream']) == 0: del ligomd.table['process_params'] # turn the known process_id binary for this insert into ascii for pid in known_proc.keys(): pid_str = "x'" for ch in pid: pid_str += "%02x" % ord(ch) pid_str += "'" known_proc[pid] = (pid_str, known_proc[pid]) # determine the locations of columns we need in the segment_definer table seg_def_cols = ligomd.table['segment_definer']['orderedcol'] ifos_col = seg_def_cols.index('ifos') name_col = seg_def_cols.index('name') vers_col = seg_def_cols.index('version') sdid_col = seg_def_cols.index('segment_def_id') # determine and remove known entries in the segment_definer table rmv_idx = [] for row_idx,row in enumerate(ligomd.table['segment_definer']['stream']): uniq_def = (row[ifos_col],row[name_col],row[vers_col]) try: seg_def_key[str(row[sdid_col])] = dmt_seg_def_dict[uniq_def] rmv_idx.append(row_idx) except KeyError: # we know nothing about this segment_definer, so query the database sql = "SELECT BLOB(segment_def_id) FROM segment_definer WHERE " sql += "creator_db = " + str(creator_db) + " AND " sql += "ifos = '" + row[ifos_col] + "' AND " sql += "name = '" + row[name_col] + "' AND " sql += "version = " + str(row[vers_col]) ligomd.curs.execute(sql) db_seg_def_id = ligomd.curs.fetchall() if len(db_seg_def_id) == 0: # this is a new segment_defintion with no existing entry dmt_seg_def_dict[uniq_def] = row[sdid_col] else: dmt_seg_def_dict[uniq_def] = db_seg_def_id[0][0] seg_def_key[str(row[sdid_col])] = dmt_seg_def_dict[uniq_def] rmv_idx.append(row_idx) # delete the necessary rows. if the table is empty, delete it newstream = [] for row_idx,row in enumerate(ligomd.table['segment_definer']['stream']): try: rmv_idx.index(row_idx) except ValueError: newstream.append(row) ligomd.table['segment_definer']['stream'] = newstream if len(ligomd.table['segment_definer']['stream']) == 0: del ligomd.table['segment_definer'] # now update the values in the xml with the values we know about for tabname in ligomd.table.keys(): table = ligomd.table[tabname] if tabname == 'process': # we do nothing to the process table pass elif tabname == 'segment' or tabname == 'segment_summary': # we need to update the process_id and the segment_def_id columns pid_col = table['orderedcol'].index('process_id') sdid_col = table['orderedcol'].index('segment_def_id') row_idx = 0 for row in table['stream']: try: repl_pid = proc_key[str(row[pid_col])] except KeyError: repl_pid = row[pid_col] try: repl_sdid = seg_def_key[str(row[sdid_col])] except KeyError: repl_sdid = row[sdid_col] row = list(row) row[pid_col] = repl_pid row[sdid_col] = repl_sdid table['stream'][row_idx] = tuple(row) row_idx += 1 else: # we just need to update the process_id column pid_col = table['orderedcol'].index('process_id') row_idx = 0 for row in table['stream']: try: repl_pid = proc_key[str(row[pid_col])] row = list(row) row[pid_col] = repl_pid table['stream'][row_idx] = tuple(row) except KeyError: pass row_idx += 1 # insert the metadata into the database logger.debug("inserting xml data") result = str(ligomd.insert()) logger.debug("insertion complete") # update the end time of known processes in the process table for pid in known_proc.keys(): # first check to see if we are backfilling missing segments sql = "SELECT end_time,domain FROM process " sql += " WHERE process_id = " + known_proc[pid][0] ligomd.curs.execute(sql) last_end_time = ligomd.curs.fetchone() # check the dn in the row we are about to update matches the users dn dn = last_end_time[1].strip() if subject != dn: msg = "\"%s\" does not match dn in existing row entries: " % subject msg += "%s (process_id %s)" % (dn, known_proc[pid][0]) logger.warn(msg) else: logger.debug('"%s" updating process_id %s' % (dn, known_proc[pid][0])) if int(known_proc[pid][1]) <= int(last_end_time[0]): logger.debug("Backfilling missing segments for process_id " + known_proc[pid][0] + " not updating end_time") else: # if we are not backfilling, update the end_time of the process sql = "UPDATE process SET end_time = " + str(known_proc[pid][1]) sql += " WHERE process_id = " + known_proc[pid][0] sql += " AND end_time < " + str(known_proc[pid][1]) ligomd.curs.execute(sql) ligomd.dbcon.commit() logger.info("Method insert: %s rows affected by insert" % result) except Exception as e: start_response("500 Internal Server Error", [('Content-type', 'text/plain')]) msg = "500 Internal Server Error\n\n%s" % e logger.error(msg) return [ msg ] try: del ligomd del lwtparser del known_proc del seg_def_key del proc_key except Exception as e: logger.error("Error deleting metadata object in method insertdmt: %s" % e) # encode the result result = cjson.encode(result) # return the result header = [('Content-Type', 'application/json')] start_response("200 OK", header) return [ result ]
def insert(self, environ, start_response): """ """ global xmlparser, dbobj logger.debug("Method insert called") # use specific grid-mapfile for insert operation mapfile = self.configuration['gridmap_insert'] # check authorization authorized, subject = self.checkAuthorizationGridMap(environ, mapfile) if not authorized: start_response("401 Unauthorized", [('Content-type', 'text/plain')]) msg = "401 Unauthorized\n\nSubject %s is not authorized for method insert" % subject logger.info("Subject %s is not authorized for method insert" % subject) return [ msg ] else: logger.info("Subject %s is authorized for method insert" % subject) # pick off the format input type format = environ['wsgiorg.routing_args'][1]['format'] # for a POST the format must be JSON if format != 'json': start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request\n\nformat must be 'json' for POST operation" return [ msg ] # read the incoming payload try: #import simplejson (moved to top) wsgiIn=environ['wsgi.input'].read() inputString=simplejson.loads(wsgiIn) #inputString = cjson.decode(environ['wsgi.input'].read()) except Exception as e: start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request" logger.debug("Error decoding input: %s" % e) return [ msg ] logger.debug("Method insert called with '%s'" % inputString) try: # create a ligo metadata object lwtparser = ldbd.LIGOLwParser() ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj) # parse the input string into a metadata object ligomd.parse(inputString) # add a gridcert table to this request containing the users dn ligomd.set_dn(subject) # insert the metadata into the database result = str(ligomd.insert()) logger.info("Method insert: %s rows affected by insert" % result) except Exception as e: start_response("500 Internal Server Error", [('Content-type', 'text/plain')]) msg = "500 Internal Server Error\n\n%s" % e logger.error(msg) return [ msg ] try: del ligomd del lwtparser except Exception as e: logger.error("Error deleting metadata object in method query: %s" % e) # encode the result result = cjson.encode(result) # return the result header = [('Content-Type', 'application/json')] start_response("200 OK", header) return [ result ]
def query(self, environ, start_response): """ """ global xmlparser, dbobj logger.debug("Method query called") # determine protocol try: protocol, querystr = (cjson.decode(environ['wsgi.input'].read())).split(":") except Exception as e: start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request" logger.debug("Error decoding input: %s" % e) return [ msg ] if protocol == "https": # use generic grid-mapfile for query operation mapfile = self.configuration['gridmap'] # check authorization authorized, subject = self.checkAuthorizationGridMap(environ, mapfile) if not authorized: start_response("401 Unauthorized", [('Content-type', 'text/plain')]) msg = "401 Unauthorized\n\nSubject %s is not authorized for method query" % subject logger.info("Subject %s is not authorized for method query" % subject) return [ msg ] else: logger.info("Subject %s is authorized for method query" % subject) else: logger.info("Method QUERY is being called from internal network") # pick off the format input type format = environ['wsgiorg.routing_args'][1]['format'] # for a POST the format must be JSON if format != 'json': start_response("400 Bad Request", [('Content-type', 'text/plain')]) msg = "400 Bad Request\n\nformat must be 'json' for POST operation" return [ msg ] logger.debug("Method query called with '%s'" % querystr) try: # create a ligo metadata object lwtparser = ldbd.LIGOLwParser() ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj) # execute the query rowcount = ligomd.select(querystr) # convert the result to xml result = ligomd.xml() logger.debug("Method query: %d rows returned" % rowcount) except Exception as e: start_response("500 Internal Server Error", [('Content-type', 'text/plain')]) msg = "500 Internal Server Error\n\n%s" % e logger.error(msg) return [ msg ] try: del ligomd del lwtparser except Exception as e: logger.error("Error deleting metadata object in method query: %s" % e) # encode the result result = cjson.encode(result) # return the result header = [('Content-Type', 'application/json')] start_response("200 OK", header) return [ result ]