def execQuery(self, qry):
'''
Executes a SQL statement, returning the results
'''
self.connect()
logger.debug('Executing query: ' + qry)
qry = qry.strip()
if not len(qry):
return []
# any error handling we should do?
#print qry
#x = raw_input()
#if x == '':
# x = qry
#self.dbconn = None
#self.dbconn.reset()
#self.connect()
#Check to see if the connection is valid
if (self.dbconn.status != 1):
logger.info("DB Connection bad, attempting reset")
self.dbconn.reset()
response = self.dbconn.query(qry)
if response: return response.getresult()
def loadharvest(self, dir=config.datadir):
'''Uses psql to stage harvested data in the quarry database'''
qlog.info("Loading harvest from %s" % (dir, ))
t = time.time()
sql = "%s;\n %s;\n %s;\n %s;" % (
"begin transaction", queries.load_resources(dir),
queries.load_descriptors(dir), "commit")
sqlfile = config.tmpdir + "/temp.sql"
f = file(sqlfile, "w")
f.write(sql)
f.close()
self.closeFiles()
cmd = '''%s -h %s -f "%s" %s''' % (config.psqlpath, self.qdb.hostname,
sqlfile, self.qdb.dbname)
flusher = popen2.Popen3(cmd)
output = flusher.fromchild
# don't quit till this process finishes
qlog.debug('psql response:\n' + output.read())
flusher.wait()
qlog.info("...bulk loaded in %s seconds." % (time.time() - t, ))
def loadharvest(self, dir=config.datadir):
'''Uses psql to stage harvested data in the quarry database'''
qlog.info("Loading harvest from %s" % (dir,))
t = time.time()
sql = "%s;\n %s;\n %s;\n %s;" % ("begin transaction",
queries.load_resources(dir),
queries.load_descriptors(dir),
"commit")
sqlfile = config.tmpdir + "/temp.sql"
f = file(sqlfile, "w")
f.write(sql)
f.close()
self.closeFiles()
cmd = '''%s -h %s -f "%s" %s''' % (config.psqlpath,
self.qdb.hostname,
sqlfile,
self.qdb.dbname)
flusher = popen2.Popen3(cmd)
output = flusher.fromchild
# don't quit till this process finishes
qlog.debug('psql response:\n' + output.read())
flusher.wait()
qlog.info("...bulk loaded in %s seconds." % (time.time() - t,))
def execQuery(self, qry):
'''
Executes a SQL statement, returning the results
'''
self.connect()
logger.debug('Executing query: ' + qry)
qry = qry.strip()
if not len(qry):
return []
# any error handling we should do?
#print qry
#x = raw_input()
#if x == '':
# x = qry
#self.dbconn = None
#self.dbconn.reset()
#self.connect()
#Check to see if the connection is valid
if (self.dbconn.status != 1):
logger.info("DB Connection bad, attempting reset")
self.dbconn.reset()
response = self.dbconn.query(qry)
if response: return response.getresult()
def DispatchNewResources(self):
qlog.debug("Dispatch New Resources")
self._RefreshUniqueSignatures()
for s in self.UniqueSignatures():
qlog.info("Computing extent for signature %s" % (s, ))
s.ComputeExtent()
# resource table is deprecated
self._InsertNewResources()
def DispatchNewResources(self):
qlog.debug("Dispatch New Resources")
self._RefreshUniqueSignatures()
for s in self.UniqueSignatures():
qlog.info("Computing extent for signature %s"%(s,))
s.ComputeExtent()
# resource table is deprecated
self._InsertNewResources()
def DeleteExistingResources(self):
qlog.debug("Deleting Existing Resources")
# need to find all the resources previously dispatched
# that are staged to be re-dispatched, and delete them
#self._DeleteFromSignatureExtents()
# resource table is deprecated
#self._DeleteResources()
pass
def DeleteExistingResources(self):
qlog.debug("Deleting Existing Resources")
# need to find all the resources previously dispatched
# that are staged to be re-dispatched, and delete them
#self._DeleteFromSignatureExtents()
# resource table is deprecated
#self._DeleteResources()
pass
def _makeTable(self):
'''
Create a table for the signature. This method is idempotent and non-destructive
'''
qlog.debug("Creating table " + self.strid)
x = self.qdb.checkTable(self.strid)
if not x:
fields = self.columns()
types = ['text']*len(fields)
self.qdb.createTable( self.strid, fields, types )
def execCommand(self, qry):
'''Executes a SQL statement, ignoring the results'''
#self.connect()
# any error handling we should do?
logger.debug('Executing command: ' + qry)
#Check to see if the connection is valid
if (self.dbconn.status != 1):
logger.info("DB Connection bad, attempting reset")
self.dbconn.reset()
result = self.dbconn.query(qry)
def execCommand(self, qry):
'''Executes a SQL statement, ignoring the results'''
#self.connect()
# any error handling we should do?
logger.debug('Executing command: ' + qry)
#Check to see if the connection is valid
if (self.dbconn.status != 1):
logger.info("DB Connection bad, attempting reset")
self.dbconn.reset()
result = self.dbconn.query(qry)
def Describe(key):
qlog.info("Describe(%s)" % (key,))
for s in sm.UniqueSignatures():
q = s.ConjunctiveQuery(conditions=[('userkey', key)], wildcard=False)
rs = db.execQuery(q);
if len(rs) != 0:
break
if len(rs) == 0:
raise ValueError("no resource '%s' was found" % (key,))
else:
qlog.debug("columns: %s" % (s.rawcolumns(),))
return zip(s.rawcolumns(), [str(x) for x in rs[0]])
def Describe(key):
qlog.info("Describe(%s)" % (key, ))
for s in sm.UniqueSignatures():
q = s.ConjunctiveQuery(conditions=[('userkey', key)], wildcard=False)
rs = db.execQuery(q)
if len(rs) != 0:
break
if len(rs) == 0:
raise ValueError("no resource '%s' was found" % (key, ))
else:
qlog.debug("columns: %s" % (s.rawcolumns(), ))
return zip(s.rawcolumns(), [str(x) for x in rs[0]])
def activate(self, target):
'''test this rule against a target, executing the script
on a match.'''
qlog.debug("Testing Rule '%s' against '%s'" % (self, target))
if self.regexp.match(target):
qlog.debug('Match')
try:
return reduce(operator.concat, [s.execute(target) for s in self.scripts])
#return self.script.execute(target)
except:
(e, v, t) = sys.exc_info()
qlog.error(str(e) + ": " + str(v))
raise e, v,t
return ''
else:
return ''
def activate(self, target):
'''test this rule against a target, executing the script
on a match.'''
qlog.debug("Testing Rule '%s' against '%s'" % (self, target))
if self.regexp.match(target):
qlog.debug('Match')
try:
return reduce(operator.concat,
[s.execute(target) for s in self.scripts])
#return self.script.execute(target)
except:
(e, v, t) = sys.exc_info()
qlog.error(str(e) + ": " + str(v))
raise e, v, t
return ''
else:
return ''
def LoadTriples(self, fname):
'''Load triples from an ASCII file. Use config.py to configure delimiters'''
# TODO: support RDF and N-triples format, at least
t = time.time()
qlog.info("Loading triples from %s" % (fname, ))
cmd = '''%s -h %s -c "%s" %s''' % (config.psqlpath, self.qdb.hostname,
queries.copy_triples(
fname, csv=""), self.qdb.dbname)
flusher = popen2.Popen3(cmd)
output = flusher.fromchild
# don't quit till this process finishes
qlog.debug('psql response:\n' + output.read())
flusher.wait()
qlog.info("...triples loaded in %s seconds." % (time.time() - t, ))
def _populateTable(self):
'''
Load a profile table from the staging data.
'''
sigid = self.strid
fields = ','.join(self.properties)
qlog.debug("Populating Table " + sigid)
crossqry = self.crosstab(fields)
# insert into profile table
scheme = self.columns()
scheme_s = ','.join(scheme)
insertqry = "INSERT INTO %s (%s) (%s)" % (sigid, scheme_s, crossqry)
self.qdb.execCommand(insertqry)
def PathValues(path,
conditions,
property,
offset=0,
limit='all',
sorted=False):
'''Retrieve values of property for resources that 1) satisfy conditions and 2)
are avilable in the path context provided.'''
raw = "PathValues(%s, %s, %s)[%s,%s]%s" % (path, conditions, property,
offset, limit, sorted)
qlog.info(raw)
try:
callstr = pattre.sub('_', raw)
start_time = time.time()
if iscached(callstr):
tuples = getCachedResults(callstr)
updateCache(callstr)
results = [t[0] for t in tuples]
else:
# ----------------
if not path:
results = ValidValues(conditions, property, offset, limit,
sorted)
else:
newpath = path + [(conditions, property)]
q = TraverseQuery(newpath)
q = PagedQuery(q, offset, limit, sorted)
tuples = db.execQuery(q)
tuples = asList(tuples)
results = [t[0] for t in tuples]
# ----------------
t = time.time() - start_time
if t > INTERACTIVE_SPEED and not iscached(callstr):
cacheQuery(callstr, results)
qlog.debug(str(results))
qlog.info("----- finished in: %f secs" % (t, ))
return results
except Exception, e:
(et, v, t) = sys.exc_info()
qlog.ExceptionMessage(et, v, t)
raise
def LoadTriples(self, fname):
'''Load triples from an ASCII file. Use config.py to configure delimiters'''
# TODO: support RDF and N-triples format, at least
t = time.time()
qlog.info("Loading triples from %s" % (fname,))
cmd = '''%s -h %s -c "%s" %s''' % (config.psqlpath,
self.qdb.hostname,
queries.copy_triples(fname, csv=""),
self.qdb.dbname)
flusher = popen2.Popen3(cmd)
output = flusher.fromchild
# don't quit till this process finishes
qlog.debug('psql response:\n' + output.read())
flusher.wait()
qlog.info("...triples loaded in %s seconds." % (time.time() - t,))
def PathValues(path, conditions, property, offset=0, limit='all', sorted=False):
'''Retrieve values of property for resources that 1) satisfy conditions and 2)
are avilable in the path context provided.'''
raw = "PathValues(%s, %s, %s)[%s,%s]%s" % (path, conditions,property,offset,limit,sorted)
qlog.info(raw)
try:
callstr = pattre.sub('_', raw)
start_time = time.time()
if iscached(callstr):
tuples = getCachedResults(callstr)
updateCache(callstr)
results = [t[0] for t in tuples]
else:
# ----------------
if not path:
results = ValidValues(conditions, property, offset, limit, sorted)
else:
newpath = path + [(conditions, property)]
q = TraverseQuery(newpath)
q = PagedQuery(q, offset, limit, sorted)
tuples = db.execQuery(q)
tuples = asList(tuples)
results = [t[0] for t in tuples]
# ----------------
t = time.time() - start_time
if t > INTERACTIVE_SPEED and not iscached(callstr):
cacheQuery(callstr, results)
qlog.debug(str(results))
qlog.info("----- finished in: %f secs" % (t,))
return results
except Exception, e:
(et,v,t) = sys.exc_info()
qlog.ExceptionMessage(et,v,t)
raise
def PathProperties(path, conds):
'''Retrieve unique properties for resources that 1) satisfy conditions and 2)
are avilable in the path context provided.'''
raw = "PathProperties(%s, %s)" % (path, conds)
qlog.info(raw)
try:
callstr = pattre.sub('_', raw)
start_time = time.time()
if iscached(callstr):
tuples = getCachedResults(callstr)
updateCache(callstr)
results = ('', [t[0] for t in tuples])
else:
# ----------------
if not path:
props = ValidProps(conds)
results = '', props
elif not conds:
results = PropertiesOf(TraverseQuery(path), True)
else:
newpath = path + [(conds, 'userkey')]
results = Traverse(newpath)
# ----------------
t = time.time() - start_time
if t > INTERACTIVE_SPEED and not iscached(callstr):
cacheQuery(callstr, results[1])
qlog.debug(str(results))
qlog.info("----- finished in: %f secs" % (t, ))
return results
except Exception, e:
(et, v, t) = sys.exc_info()
qlog.ExceptionMessage(et, v, t)
raise
def PathProperties(path, conds):
'''Retrieve unique properties for resources that 1) satisfy conditions and 2)
are avilable in the path context provided.'''
raw = "PathProperties(%s, %s)" % (path, conds)
qlog.info(raw)
try:
callstr = pattre.sub('_', raw)
start_time = time.time()
if iscached(callstr):
tuples = getCachedResults(callstr)
updateCache(callstr)
results = ('', [t[0] for t in tuples])
else:
# ----------------
if not path:
props = ValidProps(conds)
results = '', props
elif not conds:
results = PropertiesOf(TraverseQuery(path), True)
else:
newpath = path + [(conds, 'userkey')]
results = Traverse(newpath)
# ----------------
t = time.time() - start_time
if t > INTERACTIVE_SPEED and not iscached(callstr):
cacheQuery(callstr, results[1])
qlog.debug(str(results))
qlog.info("----- finished in: %f secs" % (t,))
return results
except Exception, e:
(et,v,t) = sys.exc_info()
qlog.ExceptionMessage(et,v,t)
raise
def ClearSignatures(self):
qlog.debug("Clearing all Signatures")
for s in self.UniqueSignatures():
s.DropExtent()
s.DeleteSignature()
def export_PathProperties(self, path, conditions):
qlog.debug("Hello?")
return quarry.PathProperties(path, conditions)
def ClearStagingArea(self):
qlog.debug("Clearing Staging Area")
self.qdb.execCommand(queries.delete_staging_area)
def ClearSignatures(self):
qlog.debug("Clearing all Signatures")
for s in self.UniqueSignatures():
s.DropExtent()
s.DeleteSignature()
def export_PathProperties(self, path, conditions):
qlog.debug("Hello?")
return quarry.PathProperties(path, conditions)
def ComputeExtent(self):
qlog.debug("Compute Extent for signature %s" % (self.properties,))
self._makeTable()
self._populateTable()
def ClearStagingArea(self):
qlog.debug("Clearing Staging Area")
self.qdb.execCommand(queries.delete_staging_area)
