def query(self, arg):
"""
Execute an SQL query on the database and return the result as LIGO_LW XML
@param arg: a text string containing an SQL query to be executed
@return: None
"""
global logger
global xmlparser, dbobj
# get the query string and log it
querystr = arg[0]
logger.debug("Method query called with %s" % querystr)
# assume failure
code = 1
try:
# create a ligo metadata object
lwtparser = ldbd.LIGOLwParser()
ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj)
# execute the query
rowcount = ligomd.select(querystr)
# convert the result to xml
result = ligomd.xml()
logger.debug("Method query: %d rows returned" % rowcount)
code = 0
except Exception, e:
result = ("Error querying metadata database: %s" % e)
logger.error(result)
def __init__(self, client): def dtd_uri_callback(uri): if uri == 'http://ldas-sw.ligo.caltech.edu/doc/ligolwAPI/html/ligolw_dtd.txt': return 'file://localhost' + os.path.join( os.environ["GLUE_PREFIX"], 'etc/ligolw_dtd.txt' ) else: return uri self.client = client self.xmlparser = pyRXP.Parser() self.xmlparser.eoCB = dtd_uri_callback self.lwtparser = ldbd.LIGOLwParser() self.ligomd = ldbd.LIGOMetadata(self.xmlparser, self.lwtparser, None)
def setupSegment_md(filename, xmlparser, lwtparser, debug):
"""
Helper function used to setup ligolw parser (S6 xml generation tool).
"""
segment_md = ldbd.LIGOMetadata(xmlparser, lwtparser)
#if debug:
# print "Inserting file %s." % filename
fh = open(filename, 'r')
xmltext = fh.read()
fh.close()
segment_md.parse(xmltext)
if debug:
#segment_md.table
segment_md.table.keys()
return segment_md
# pick off the format input type
format = environ['wsgiorg.routing_args'][1]['format']
# for a POST the format must be JSON
if format != 'json':
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request\n\nformat must be 'json' for POST operation"
return [msg]
logger.debug("Method query called with '%s'" % querystr)
try:
# create a ligo metadata object
lwtparser = ldbd.LIGOLwParser()
ligomd = ldbd.LIGOMetadata(xmlparser, lwtparser, dbobj)
# execute the query
rowcount = ligomd.select(querystr)
# convert the result to xml
result = ligomd.xml()
logger.debug("Method query: %d rows returned" % rowcount)
except Exception, e:
start_response("500 Internal Server Error",
[('Content-type', 'text/plain')])
msg = "500 Internal Server Error\n\n%s" % e
logger.error(msg)
return [msg]
def insertdmt(self, environ, start_response):
"""
"""
global xmlparser, dbobj
global dmt_proc_dict, dmt_seg_def_dict, creator_db
proc_key = {}
known_proc = {}
seg_def_key = {}
logger.debug( "Method dmtinsert called." )
logger.debug( "Known processes %s, " % str(dmt_proc_dict) )
logger.debug( "Known segment_definers %s" % str(dmt_seg_def_dict) )
# use specific grid-mapfile for insertdmt operation
mapfile = self.configuration['gridmap_insertdmt']
# check authorization
authorized, subject = self.checkAuthorizationGridMap(environ, mapfile)
if not authorized:
start_response("401 Unauthorized", [('Content-type', 'text/plain')])
msg = "401 Unauthorized\n\nSubject %s is not authorized for method insertdmt" % subject
logger.info("Subject %s is not authorized for method insertdmt" % subject)
return [ msg ]
else:
logger.info("Subject %s is authorized for method insertdmt" % subject)
# pick off the format input type
format = environ['wsgiorg.routing_args'][1]['format']
# for a POST the format must be JSON
if format != 'json':
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request\n\nformat must be 'json' for POST operation"
return [ msg ]
# read the incoming payload
try:
inputString = cjson.decode(environ['wsgi.input'].read())
except Exception as e:
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request"
logger.debug("Error decoding input: %s" % e)
return [ msg ]
logger.debug("Method insertdmt called with '%s'" % inputString)
try:
# create a ligo metadata object
lwtparser = ldbd.LIGOLwParser()
ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj)
# parse the input string into a metadata object
logger.debug("parsing xml data")
ligomd.parse(inputString)
# store the users dn in the process table
ligomd.set_dn(subject)
# determine the local creator_db number
if creator_db is None:
sql = "SELECT DEFAULT FROM SYSCAT.COLUMNS WHERE "
sql += "TABNAME = 'PROCESS' AND COLNAME = 'CREATOR_DB'"
ligomd.curs.execute(sql)
creator_db = ligomd.curs.fetchone()[0]
# determine the locations of columns we need in the process table
process_cols = ligomd.table['process']['orderedcol']
node_col = process_cols.index('node')
prog_col = process_cols.index('program')
upid_col = process_cols.index('unix_procid')
start_col = process_cols.index('start_time')
end_col = process_cols.index('end_time')
pid_col = process_cols.index('process_id')
# determine and remove known entries from the process table
rmv_idx = []
for row_idx,row in enumerate(ligomd.table['process']['stream']):
uniq_proc = (row[node_col],row[prog_col],row[upid_col],row[start_col])
try:
proc_key[str(row[pid_col])] = dmt_proc_dict[uniq_proc]
known_proc[str(dmt_proc_dict[uniq_proc])] = row[end_col]
rmv_idx.append(row_idx)
except KeyError:
# we know nothing about this process, so query the database
sql = "SELECT BLOB(process_id) FROM process WHERE "
sql += "creator_db = " + str(creator_db) + " AND "
sql += "node = '" + row[node_col] + "' AND "
sql += "program = '" + row[prog_col] + "' AND "
sql += "unix_procid = " + str(row[upid_col]) + " AND "
sql += "start_time = " + str(row[start_col])
ligomd.curs.execute(sql)
db_proc_ids = ligomd.curs.fetchall()
if len(db_proc_ids) == 0:
# this is a new process with no existing entry
dmt_proc_dict[uniq_proc] = row[pid_col]
elif len(db_proc_ids) == 1:
# the process_id exists in the database so use that insted
dmt_proc_dict[uniq_proc] = db_proc_ids[0][0]
proc_key[str(row[pid_col])] = dmt_proc_dict[uniq_proc]
known_proc[str(dmt_proc_dict[uniq_proc])] = row[end_col]
rmv_idx.append(row_idx)
else:
# multiple entries for this process, needs human assistance
raise ServerHandlerException("multiple entries for dmt process")
# delete the duplicate processs rows and clear the table if necessary
newstream = []
for row_idx,row in enumerate(ligomd.table['process']['stream']):
try:
rmv_idx.index(row_idx)
except ValueError:
newstream.append(row)
ligomd.table['process']['stream'] = newstream
if len(ligomd.table['process']['stream']) == 0:
del ligomd.table['process']
# delete the duplicate process_params rows and clear the table if necessary
# (the DMT does not write a process_params table, so check for one first)
if 'process_params' in ligomd.table:
ppid_col = ligomd.table['process_params']['orderedcol'].index('process_id')
newstream = []
for row_idx,row in enumerate(ligomd.table['process_params']['stream']):
# if the process_id in this row is known, delete (i.e. don't copy) it
try:
proc_key[str(row[ppid_col])]
except KeyError:
newstream.append(row)
ligomd.table['process_params']['stream'] = newstream
if len(ligomd.table['process_params']['stream']) == 0:
del ligomd.table['process_params']
# turn the known process_id binary for this insert into ascii
for pid in known_proc.keys():
pid_str = "x'"
for ch in pid:
pid_str += "%02x" % ord(ch)
pid_str += "'"
known_proc[pid] = (pid_str, known_proc[pid])
# determine the locations of columns we need in the segment_definer table
seg_def_cols = ligomd.table['segment_definer']['orderedcol']
ifos_col = seg_def_cols.index('ifos')
name_col = seg_def_cols.index('name')
vers_col = seg_def_cols.index('version')
sdid_col = seg_def_cols.index('segment_def_id')
# determine and remove known entries in the segment_definer table
rmv_idx = []
for row_idx,row in enumerate(ligomd.table['segment_definer']['stream']):
uniq_def = (row[ifos_col],row[name_col],row[vers_col])
try:
seg_def_key[str(row[sdid_col])] = dmt_seg_def_dict[uniq_def]
rmv_idx.append(row_idx)
except KeyError:
# we know nothing about this segment_definer, so query the database
sql = "SELECT BLOB(segment_def_id) FROM segment_definer WHERE "
sql += "creator_db = " + str(creator_db) + " AND "
sql += "ifos = '" + row[ifos_col] + "' AND "
sql += "name = '" + row[name_col] + "' AND "
sql += "version = " + str(row[vers_col])
ligomd.curs.execute(sql)
db_seg_def_id = ligomd.curs.fetchall()
if len(db_seg_def_id) == 0:
# this is a new segment_defintion with no existing entry
dmt_seg_def_dict[uniq_def] = row[sdid_col]
else:
dmt_seg_def_dict[uniq_def] = db_seg_def_id[0][0]
seg_def_key[str(row[sdid_col])] = dmt_seg_def_dict[uniq_def]
rmv_idx.append(row_idx)
# delete the necessary rows. if the table is empty, delete it
newstream = []
for row_idx,row in enumerate(ligomd.table['segment_definer']['stream']):
try:
rmv_idx.index(row_idx)
except ValueError:
newstream.append(row)
ligomd.table['segment_definer']['stream'] = newstream
if len(ligomd.table['segment_definer']['stream']) == 0:
del ligomd.table['segment_definer']
# now update the values in the xml with the values we know about
for tabname in ligomd.table.keys():
table = ligomd.table[tabname]
if tabname == 'process':
# we do nothing to the process table
pass
elif tabname == 'segment' or tabname == 'segment_summary':
# we need to update the process_id and the segment_def_id columns
pid_col = table['orderedcol'].index('process_id')
sdid_col = table['orderedcol'].index('segment_def_id')
row_idx = 0
for row in table['stream']:
try:
repl_pid = proc_key[str(row[pid_col])]
except KeyError:
repl_pid = row[pid_col]
try:
repl_sdid = seg_def_key[str(row[sdid_col])]
except KeyError:
repl_sdid = row[sdid_col]
row = list(row)
row[pid_col] = repl_pid
row[sdid_col] = repl_sdid
table['stream'][row_idx] = tuple(row)
row_idx += 1
else:
# we just need to update the process_id column
pid_col = table['orderedcol'].index('process_id')
row_idx = 0
for row in table['stream']:
try:
repl_pid = proc_key[str(row[pid_col])]
row = list(row)
row[pid_col] = repl_pid
table['stream'][row_idx] = tuple(row)
except KeyError:
pass
row_idx += 1
# insert the metadata into the database
logger.debug("inserting xml data")
result = str(ligomd.insert())
logger.debug("insertion complete")
# update the end time of known processes in the process table
for pid in known_proc.keys():
# first check to see if we are backfilling missing segments
sql = "SELECT end_time,domain FROM process "
sql += " WHERE process_id = " + known_proc[pid][0]
ligomd.curs.execute(sql)
last_end_time = ligomd.curs.fetchone()
# check the dn in the row we are about to update matches the users dn
dn = last_end_time[1].strip()
if subject != dn:
msg = "\"%s\" does not match dn in existing row entries: " % subject
msg += "%s (process_id %s)" % (dn, known_proc[pid][0])
logger.warn(msg)
else:
logger.debug('"%s" updating process_id %s' % (dn, known_proc[pid][0]))
if int(known_proc[pid][1]) <= int(last_end_time[0]):
logger.debug("Backfilling missing segments for process_id " +
known_proc[pid][0] + " not updating end_time")
else:
# if we are not backfilling, update the end_time of the process
sql = "UPDATE process SET end_time = " + str(known_proc[pid][1])
sql += " WHERE process_id = " + known_proc[pid][0]
sql += " AND end_time < " + str(known_proc[pid][1])
ligomd.curs.execute(sql)
ligomd.dbcon.commit()
logger.info("Method insert: %s rows affected by insert" % result)
except Exception as e:
start_response("500 Internal Server Error", [('Content-type', 'text/plain')])
msg = "500 Internal Server Error\n\n%s" % e
logger.error(msg)
return [ msg ]
try:
del ligomd
del lwtparser
del known_proc
del seg_def_key
del proc_key
except Exception as e:
logger.error("Error deleting metadata object in method insertdmt: %s" % e)
# encode the result
result = cjson.encode(result)
# return the result
header = [('Content-Type', 'application/json')]
start_response("200 OK", header)
return [ result ]
def insert(self, environ, start_response):
"""
"""
global xmlparser, dbobj
logger.debug("Method insert called")
# use specific grid-mapfile for insert operation
mapfile = self.configuration['gridmap_insert']
# check authorization
authorized, subject = self.checkAuthorizationGridMap(environ, mapfile)
if not authorized:
start_response("401 Unauthorized", [('Content-type', 'text/plain')])
msg = "401 Unauthorized\n\nSubject %s is not authorized for method insert" % subject
logger.info("Subject %s is not authorized for method insert" % subject)
return [ msg ]
else:
logger.info("Subject %s is authorized for method insert" % subject)
# pick off the format input type
format = environ['wsgiorg.routing_args'][1]['format']
# for a POST the format must be JSON
if format != 'json':
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request\n\nformat must be 'json' for POST operation"
return [ msg ]
# read the incoming payload
try:
#import simplejson (moved to top)
wsgiIn=environ['wsgi.input'].read()
inputString=simplejson.loads(wsgiIn)
#inputString = cjson.decode(environ['wsgi.input'].read())
except Exception as e:
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request"
logger.debug("Error decoding input: %s" % e)
return [ msg ]
logger.debug("Method insert called with '%s'" % inputString)
try:
# create a ligo metadata object
lwtparser = ldbd.LIGOLwParser()
ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj)
# parse the input string into a metadata object
ligomd.parse(inputString)
# add a gridcert table to this request containing the users dn
ligomd.set_dn(subject)
# insert the metadata into the database
result = str(ligomd.insert())
logger.info("Method insert: %s rows affected by insert" % result)
except Exception as e:
start_response("500 Internal Server Error", [('Content-type', 'text/plain')])
msg = "500 Internal Server Error\n\n%s" % e
logger.error(msg)
return [ msg ]
try:
del ligomd
del lwtparser
except Exception as e:
logger.error("Error deleting metadata object in method query: %s" % e)
# encode the result
result = cjson.encode(result)
# return the result
header = [('Content-Type', 'application/json')]
start_response("200 OK", header)
return [ result ]
def query(self, environ, start_response):
"""
"""
global xmlparser, dbobj
logger.debug("Method query called")
# determine protocol
try:
protocol, querystr = (cjson.decode(environ['wsgi.input'].read())).split(":")
except Exception as e:
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request"
logger.debug("Error decoding input: %s" % e)
return [ msg ]
if protocol == "https":
# use generic grid-mapfile for query operation
mapfile = self.configuration['gridmap']
# check authorization
authorized, subject = self.checkAuthorizationGridMap(environ, mapfile)
if not authorized:
start_response("401 Unauthorized", [('Content-type', 'text/plain')])
msg = "401 Unauthorized\n\nSubject %s is not authorized for method query" % subject
logger.info("Subject %s is not authorized for method query" % subject)
return [ msg ]
else:
logger.info("Subject %s is authorized for method query" % subject)
else:
logger.info("Method QUERY is being called from internal network")
# pick off the format input type
format = environ['wsgiorg.routing_args'][1]['format']
# for a POST the format must be JSON
if format != 'json':
start_response("400 Bad Request", [('Content-type', 'text/plain')])
msg = "400 Bad Request\n\nformat must be 'json' for POST operation"
return [ msg ]
logger.debug("Method query called with '%s'" % querystr)
try:
# create a ligo metadata object
lwtparser = ldbd.LIGOLwParser()
ligomd = ldbd.LIGOMetadata(xmlparser,lwtparser,dbobj)
# execute the query
rowcount = ligomd.select(querystr)
# convert the result to xml
result = ligomd.xml()
logger.debug("Method query: %d rows returned" % rowcount)
except Exception as e:
start_response("500 Internal Server Error", [('Content-type', 'text/plain')])
msg = "500 Internal Server Error\n\n%s" % e
logger.error(msg)
return [ msg ]
try:
del ligomd
del lwtparser
except Exception as e:
logger.error("Error deleting metadata object in method query: %s" % e)
# encode the result
result = cjson.encode(result)
# return the result
header = [('Content-Type', 'application/json')]
start_response("200 OK", header)
return [ result ]
