def parseProduction(parser, lhs, tok, stream):
"The parser itself."
if tok() is None: return None
name, thing, line = tok()
lookupTable = parser.branchTable[lhs]
rhs = lookupTable.get(name, None) # Predict branch from token
if rhs == None:
progress("""Found %s when expecting some form of %s,
\tsuch as %s\n\t%s""" % (tok(), lhs, lookupTable.keys(), parser.around(None, None)))
raise SyntaxError("""Found %s when expecting some form of %s,
\tsuch as %s\n\t%s""" % (tok(), lhs, lookupTable.keys(), parser.around(None, None)))
if parser.verb: progress( "%i %s means expand %s as %s" %(parser.lineNumber,tok(), lhs, rhs.value()))
tree = [lhs]
for term in rhs:
lit = term.fragid
if lit != name: # Not token
if lit in parser.tokenSet:
progress("Houston, we have a problem. %s is not equal to %s" % (lit, name))
progress("recursing on %s, which is not %s. Token is %s" % (lit, name, `tok()`))
tree.append(parser.parseProduction(term, tok, stream))
else:
progress("We found %s, which matches %s" % (lit, `tok()`))
tree.append(tok())
tok(parser.token(stream)) # Next token
if tok():
name, thing, line = tok()
else:
name, thing = None, None
if hasattr(parser, "p_" + lhs.fragid):
return getattr(parser, "p_" + lhs.fragid)(tree)
return tree
def internalCheck():
global kb
global cat
transactions = kb.each(pred=rdf.type, obj=cat_ns.Internal)
unbalanced = []
while len(transactions) > 0:
x = transactions.pop()
month = monthNumber(x)
if month < 0 : continue
date = str(kb.the(subj=x, pred=qu.date))
if len(kb.each(subj=x, pred=qu.in_USD)) != 1:
progress("Ignoring !=1 amount transaction %s" % x)
continue
amount = float(str(kb.the(subj=x, pred=qu.in_USD)))
for y in transactions:
datey = str(kb.the(subj=y, pred=qu.date))
if 1: # date[0:10] == datey[0:10]: # precision one day
usds = kb.each(subj=y, pred=qu.in_USD)
if len(usds) == 0:continue # No amount => must be not in this period.
if len(usds) != 1:
progress("Error: Ignoring: %i != 1 USD amounts for Internal transaction %s" % (len(usds), `y`+': '+ `usds`))
transactions.remove(y)
continue
if abs(amount +
float(str(kb.the(subj=y, pred=qu.in_USD)))) < 0.001:
transactions.remove(y)
break
else:
unbalanced.append(x)
if unbalanced:
print "<h2>Unbalanced internal transactions</h2>"
print transactionTable(unbalanced);
return
def loadFiles(files):
graph = myStore.formula()
graph.setClosureMode("e") # Implement sameAs by smushing
if verbose>0: progress("Loading %s..." % files)
graph = myStore.loadMany(files, openFormula=graph)
if verbose>0: progress("Loaded", graph)
return graph
def saveAs(uri, filename):
gifStream = urlopen(uri)
gifData = gifStream.read()
gifStream.close
progress('curl "%s" > %s' % (uri, filename))
saveStream = open(filename, "w")
saveStream.write(gifData)
saveStream.close()
def getSize(s, atr):
i = s.find(atr + '="') + len(atr) + 2
val = ""
while s[i] in "0123456789":
val += s[i]
i = i + 1
x = int(val)
progress("Found attribute %s=%i" % (atr, x))
return x
def token(parser, str, i):
"""The Tokenizer: returns (token type character, offset of token)
Skips spaces.
"0" means numeric
"a" means alphanumeric
"""
while 1:
m = whiteSpace.match(str, i)
if m == None or m.end() == i: break
i = m.end()
parser.countLines(str, i)
if i == len(str):
return "", i # eof
if parser.verb: progress( "%i) Looking at: ...%s$%s..." % (
parser.lineNumber, str[i-10:i],str[i:i+10]))
for double in "=>", "<=", "^^":
if double == str[i:i+2]: return double, i
ch = str[i]
if ch == ".": parser.keywordMode = 0 # hack
if ch in singleCharacterSelectors:
return ch, i
if ch in "+-0123456789":
return "0", i # Numeric
j = i+1
if ch == "@":
if i!=0 and whiteSpace.match(str[i-1]).end() == 0:
return ch, i
while str[j] not in notNameChars: j = j + 1
if str[i+1:j] == "keywords" :
parser.keywords = [] # Special
parser.keywordMode = 1
return str[i:j], i # keyword
if ch == '"': #"
return '"', i #"
# Alphanumeric: keyword hacks
while str[j] not in notQNameChars: j = j+1
word = str[i:j]
if parser.keywordMode:
parser.keywords.append(word)
elif word in parser.keywords:
if word == "keywords" :
parser.keywords = [] # Special
parser.keywordMode = 1
if parser.atMode:
return "@" + word, i # implicit keyword
return word, i
return "a", i # qname, langcode, or barename
def removeCommon(f, g, match):
"""Find common statements from f and g
match gives the dictionary mapping bnodes in f to bnodes in g"""
only_f, common_g = Set(), Set()
for st in f.statements[:]:
s, p, o = st.spo()
assert s not in f._redirections
assert o not in f._redirections
if s.generated(): sg = match.get(s, None)
else: sg = s
if o.generated(): og = match.get(o, None)
else: og = o
if og != None and sg != None:
gsts = g.statementsMatching(subj=sg, pred=p, obj=og)
if len(gsts) == 1:
if verbose>4: progress("Statement in both", st)
common_g.add(gsts[0])
continue
only_f.add(st)
return only_f, Set(g.statements)-common_g
def consolidate(delta, patchVerb):
"""Consolidate patches
Where the same left hand side applies to more than 1 RHS formula,
roll those RHS formulae into one, to make the dif file more readable
and faster to execute in some implementations
"""
agenda = {}
if verbose >3: progress("Consolidating %s" % patchVerb)
for s in delta.statementsMatching(pred=patchVerb):
list = agenda.get(s.subject(), None)
if list == None:
list = []
agenda[s.subject()] = list
list.append(s)
for lhs, list in agenda.items():
if verbose >3: progress("Patches lhs= %s: %s" %(lhs, list))
if len(list) > 1:
rhs = delta.newFormula()
for s in list:
delta.store.copyFormula(s.object(), rhs)
delta.removeStatement(s)
delta.add(subj=lhs, pred=patchVerb, obj=rhs.close())
def parseProduction(parser, lhs, str, tok=None, here=0):
"The parser itself."
if tok == "": return tok, here # EOF
lookupTable = parser.branchTable[lhs]
rhs = lookupTable.get(tok, None) # Predict branch from token
if rhs == None:
progress("""Found %s when expecting some form of %s,
\tsuch as %s\n\t%s""" % (tok, lhs, lookupTable.keys(), parser.around(str, here)))
raise SyntaxError("""Found %s when expecting some form of %s,
\tsuch as %s\n\t%s""" % (tok, lhs, lookupTable.keys(), parser.around(str, here)))
if parser.verb: progress( "%i %s means expand %s as %s" %(parser.lineNumber,tok, lhs, rhs.value()))
for term in rhs:
if isinstance(term, Literal): # CFG Terminal
lit = term.value()
next = here + len(lit)
if str[here:next] == lit: pass
elif "@"+str[here:next-1] == lit: next = next-1
else: raise SyntaxError(
"Found %s where %s expected\n\t %s" %
(`str[here:next]`, lit, parser.around(str, here)))
else:
rexp = tokenRegexps.get(term, None)
if rexp == None: # Not token
tok, here = parser.parseProduction(term, str, tok, here)
continue
m = rexp.match(str, here)
if m == None:
progress("\n\n\nToken: should match %s\n\t %s" %
(rexp.pattern, parser.around(str, here)))
raise SyntaxError("Token: should match %s\n\t %s" %
(rexp.pattern, parser.around(str, here)))
if parser.verb: progress( "Token matched to <%s> as pattern <%s>" % (str[here:m.end()], rexp.pattern))
next = m.end()
tok, here = parser.token(str, next) # Next token
return tok, here
def match(s, extras=BindingTree()):
"""
Input: an intermediate state s
Output: the mapping between the two graphs
When a match forces a predicate match, we add that
to extras --- we go through all of those before continuing
on our regularly scheduled P(s)
"""
progress("starting match")
progress("s.map=%s" % s.map)
G2 = s.problem.G2
for choice in extras:
if not choice:
if set(s.map.values()) >= G2.allNodes():
yield s.map
elif set(s.map.values()) >= G2.nodes():
yield finish(s, s.map)
nodeList = P(s)
else:
n, m = choice[0]
nodeList = [(n, m, choice[1:])]
nodeList = [x for x in nodeList]
progress("nodeList=", nodeList)
for n, m, realExtras in nodeList:
progress("... trying n,m=%s,%s" % (n, m))
newExtras = BindingTree()
newExtras.int_and(realExtras)
if F(s, n, m, newExtras):
s2 = s.addNode(n, m)
for x in match(s2, newExtras):
yield x
s2.undo()
def P(s):
"""
Input: a state s
Output: possible pairs to add to the mapping
"""
G1 = s.problem.G1
G2 = s.problem.G2
t1_out_size, t2_out_size, t1_in_size, t2_in_size = (len(s.t1_out), len(s.t2_out), len(s.t1_in), len(s.t2_in))
progress("P(s) %s %s %s %s" % (t1_out_size, t2_out_size, t1_in_size, t2_in_size))
if t1_out_size and t2_out_size:
progress(", case 1")
m = s.t2_out.first()
if representsSelf(m):
if m in s.t1_out:
yield m, m, regular
else:
for n in s.t1_out:
yield n, m, regular
elif not t1_out_size and not t2_out_size and t1_in_size and t2_in_size:
progress(", case 2")
m = s.t2_in.first()
if representsSelf(m):
if m in s.t1_in:
yield m, m, regular
else:
for n in s.t1_in:
yield n, m, regular
elif not t1_out_size and not t2_out_size and not t1_in_size and not t2_in_size:
progress(", case 3")
m = s.G2_not_taken.first()
if representsSelf(m):
if m in s.G1_not_taken:
yield m, m, regular
else:
for n in s.G1_not_taken:
yield n, m, regular
def lookUp(predicates, assumptions=Set()):
"""Look up all the schemas for the predicates given"""
global verbose
schemas = assumptions
for pred in predicates:
if verbose > 3: progress("Predicate: %s" % `pred`)
u = pred.uriref()
hash = u.find("#")
if hash <0:
if verbose > 1: progress("Warning: Predicate <%s> looks like web resource not Property" % u)
else:
schemas.add(u[:hash])
if verbose > 2:
for r in schemas:
progress("Metadata to be loaded: ", r)
if schemas:
return loadMany([(x) for x in schemas])
return myStore.store.newFormula() # Empty formula
def doCommand(serialDevice=None, outputURI=None, doTracks=1, doWaypoints=1, verbose=0):
if os.name == 'nt':
if not serialDevice: serialDevice = "com1"
phys = Win32SerialLink(serialDevice)
else:
if not serialDevice: serialDevice = "/dev/ttyS0"
# serialDevice = "/dev/cu.USA19H191P1.1"
phys = UnixSerialLink(serialDevice)
gps = Garmin(phys)
print "GPS Product ID: %d Descriptions: %s Software version: %2.2f" % \
(gps.prod_id, gps.prod_descs, gps.soft_ver)
f = formula() # Empty store of RDF data
base = uripath.base()
record = f.newBlankNode()
f.add(record, RDF.type, GPS.Record)
if doWaypoints:
# show waypoints
if verbose: print "Getting waypoints"
wpts = gps.getWaypoints()
for w in wpts:
if verbose: progress(`w`)
wpt = symbol(uripath.join(base, w.ident))
f.add(record, GPS.waypoint, wpt)
f.add(wpt, WGS.lat, obj=intern(degrees(w.slat)))
f.add(wpt, WGS.long, obj=intern(degrees(w.slon)))
if doTracks:
# show track
if verbose: print "Getting tracks"
tracks = gps.getTracks()
for t in tracks:
track = f.newBlankNode()
f.add(record, GPS.track, track)
for p in t:
if isinstance(p, TrackHdr):
if verbose: progress(`p`)
f.add(track, GPS.disp, intern(p.dspl))
f.add(track, GPS.color, intern(p.color))
f.add(track, GPS.trk_ident, intern(p.trk_ident))
else:
if verbose: progress(`p`)
point = f.newBlankNode()
f.add(track, GPS.trackpoint, point)
f.add(point, WGS.lat, obj=intern(degrees(p.slat)))
f.add(point, WGS.long, obj=intern(degrees(p.slon)))
# if verbose: progress(" time=", p.time)
# progress('p.time='+`p.time`) # @@
if p.time == 0 or p.time == 0xffffffffL:
if verbose: progress("time=%8x, ignoring" % p.time)
else:
f.add(point, WGS.time, obj=intern(isodate.fullString(TimeEpoch+p.time)))
phys.f.close() # Should really be done by the del() below, but isn't
del(phys) # close serial link (?)
f = f.close()
if verbose:
progress("Beginning output. You can disconnect the GPS now.")
s = f.n3String(base=base, flags="d") # Flag - no default prefix, preserve gps: prefix hint
if outputURI != None:
op = open(outputURI, "w")
op.write(s)
op.close()
else:
print s
def fyi(str, level=0, thresh=50):
if chatty >= thresh:
if isinstance(str, (lambda: True).__class__):
str = str()
progress(" " * (level * 4), str)
return None
def doProduction(lhs):
"Generate branch tables for one production"
global branchTable
if lhs is BNF.void:
progress("\nvoid")
return
if lhs is BNF.eof:
progress( "\nEOF")
return
if isinstance(lhs, Literal):
literalTerminals[lhs.value()] = 1
return
branchDict = {}
rhs = g.the(pred=BNF.matches, subj=lhs)
if rhs != None:
if chatty_flag: progress( "\nToken %s matches regexp %s" %(lhs, rhs))
try:
tokenRegexps[lhs] = re.compile(rhs.value(), re.U)
except:
print rhs.value().encode('utf-8')
raise
cc = g.each(subj=lhs, pred=BNF.canStartWith)
if cc == []: progress (recordError(
"No record of what token %s can start with" % `lhs`))
if chatty_flag: progress("\tCan start with: %s" % cc)
return
if g.contains(subj=lhs, pred=RDF.type, obj=REGEX.Regex):
import regex
rhs = regex.makeRegex(g, lhs)
try:
tokenRegexps[lhs] = re.compile(rhs, re.U)
except:
print rhs
raise
cc = g.each(subj=lhs, pred=BNF.canStartWith)
if cc == []: progress (recordError(
"No record of what token %s can start with" % `lhs`))
if chatty_flag: progress("\tCan start with: %s" % cc)
return
rhs = g.the(pred=BNF.mustBeOneSequence, subj=lhs)
if rhs == None:
progress (recordError("No definition of " + `lhs`))
return
# raise RuntimeError("No definition of %s in\n %s" %(`lhs`, `g`))
options = rhs
if chatty_flag: progress ( "\nProduction %s :: %s ie %s" %(`lhs`, `options` , `options.value()`))
succ = g.each(subj=lhs, pred=BNF.canPrecede)
if chatty_flag: progress("\tCan precede ", succ)
branches = g.each(subj=lhs, pred=BNF.branch)
for branch in branches:
option = g.the(subj=branch, pred=BNF.sequence)
if chatty_flag: progress( "\toption: "+`option.value()`)
for part in option:
if part not in already and part not in agenda: agenda.append(part)
y = `part`
conditions = g.each(subj=branch, pred=BNF.condition)
if conditions == []:
progress(
recordError(" NO SELECTOR for %s option %s ie %s" %
(`lhs`, `option`, `option.value()` )))
if option.value == []: # Void case - the tricky one
succ = g.each(subj=lhs, pred=BNF.canPrecede)
for y in succ:
if chatty_flag: progress("\t\t\tCan precede ", `y`)
if chatty_flag: progress("\t\tConditions: %s" %(conditions))
for str1 in conditions:
if str1 in branchDict:
progress(recordError(
"Conflict: %s is also the condition for %s" % (
str1, branchDict[str1].value())))
branchDict[str1.__str__()] = option
# break
for str1 in branchDict:
for str2 in branchDict:
s1 = unicode(str1)
s2 = unicode(str2)
# @@ check that selectors are distinct, not substrings
if (s1.startswith(s2) or s2.startswith(s1)) and branchDict[str1] is not branchDict[str2]:
progress("WARNING: for %s, %s indicates %s, but %s indicates %s" % (
lhs, s1, branchDict[str1], s2, branchDict[str2]))
branchTable[lhs] = branchDict
def __init__(self, s, level=0):
self._s = s
if True or chatty > 0:
progress(" " * (level * 4), "Proof failed: ", s)
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
"""Fin - financial summary
<command> <options> <inputURIs>
Totals transactions by classes to which they are known to belong
This is or was http://www.w3.org/2000/10/swap/pim/fin.py
"""
#import urllib
#import time
import sys
# global sax2rdf
global kb, tax
def noteError(e):
if not errors.get(s, None): errors[s] = [];
errors[s].append(e)
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
fatalErrors = 0;
# Load the data:
kb = loadMany(inputURIs)
qu_date = qu.date
qu_in_USD = qu.in_USD
qu_amount = qu.amount
qu_payee = qu.payee
qu_Classified = qu.Classified
qu_Unclassified = qu.Unclassified
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
if verbose:
progress("Tax categories" + `taxCategories`)
specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]
####### Analyse the data:
numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
monthTotals = [0] * numberOfMonths
incomeByMonth = [0] * numberOfMonths
income, outgoings = 0,0
outgoingsByMonth = [0] * numberOfMonths
quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
bottomCategories = [];
for c in quCategories:
if isBottomClass(c): bottomCategories.append(c);
totals = {} # Total by all classes of transaction
count = {} # Number of transactions
byMonth = {}
sts = kb.statementsMatching(pred=qu.amount) # Ideally one per transaction
errors = {}
for st in sts:
s = st.subject()
uri = s.uriref()
# classified = kb.each(pred=rdf_type, obj=qu_Classified)
# unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
# for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
# for s in classified + unclassified:
# progress( "Transaction ", `s`)
t_ok, c_ok = 0, 0
cats = allClasses(kb.each(subj=s, pred=rdf.type))
# progress( "Categories: "+`cats`)
month = monthNumber(s)
if month not in range(numberOfMonths) : continue
payees = kb.each(subj=s, pred=qu_payee)
if not payees:
progress("@@@ Error: No payee for "+`uri`)
payee = "@@@@@@ Unknown";
fatalErrors += 1;
elif len(payees) >1 and str(payees[0]) == "Check":
payee = payees[1]
else:
payee = payees[0]
amounts = kb.each(subj=s, pred=qu_in_USD)
if len(amounts) == 0:
amounts = kb.each(subj=s, pred=qu_amount)
if len(amounts) == 0:
progress("@@@ Error: No USD amount for "+`uri`)
fatalErrors += 1;
else:
progress("Warning: No USD amount for "+`uri`+", assuming USD")
if len(amounts) >1:
if (cat_ns.Internal not in cats or
len(amounts) != 2 ):
fatalErrors += 1;
progress(
"Error: More than one USD amount %s for transaction %s -- ignoring!\n"
% (`amounts`,uri))
else:
sum = float(amounts[0]) + float(amounts[1])
if sum != 0:
fatalErrors += 1;
progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
% (sum, amounts, uri))
continue
if len(amounts) != 1:
progress("@@@ Error: No amount for "+`uri`);
fatalErrors += 1;
ss = kb.statementsMatching(subj=s)
progress(`ss`+'; KB='+`kb.n3String()`)
continue
amount = float(amounts[0].__str__())
# print "%s %40s %10s month %i" %(date, payee, `amount`, month)
monthTotals[month] = monthTotals[month] + amount
if cat_ns.Internal not in cats:
if amount > 0:
incomeByMonth[month] = incomeByMonth[month] + amount
income = income + amount
else:
outgoingsByMonth[month] = outgoingsByMonth[month] + amount
outgoings = outgoings + amount
normalCats = [] # For this item
for c in cats:
totals[c] = totals.get(c, 0) + amount
byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
count[c] = count.get(c, 0) + 1
byMonth[c][month] = byMonth[c][month] + amount
if c not in specialCategories:
normalCats.append(c)
bottomCats = normalCats[:] # Copy
for b in normalCats:
sups = kb.each(subj=b, pred=rdfs.subClassOf)
for sup in sups:
if sup in bottomCats:
bottomCats.remove(sup)
if len(bottomCats) == 0:
noteError("No categoriy: %s for <%s>" # all cats: %s, raw cats:%s"
%(`cats`, `s`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
noteError("Be more specifc: %s for <%s>" %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
if len(bottomCats) > 1:
noteError("Inconsistent categories: %s" # all cats: %s, raw cats:%s"
%(`bottomCats`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
print '<html xmlns="http://www.w3.org/1999/xhtml">'
if '--summary' in sys.argv:
title = "Monthly summary"
elif '--issues' in sys.argv:
title = "Issues"
else:
title = "Report"
print """<head>
<meta charset='UTF-8'>
<title>%s</title>
<link rel="Stylesheet" href="report.css">
</head>
<body>
""" % (title)
# <img src="sand-dollar.gif" alt="dollar" align="right"/>
version = "$Id$"
# SUMMARY TABLE OF CATEGORY BY MONTH
if '--summary' in sys.argv:
print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
print "<tr><th></th><th>Total </th>"
for month in range(numberOfMonths):
m = month + int(startDate[5:7]) - 1
while m > 11: m -= 12 # Modulo in python?
print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
print "</tr>"
def listFor(c, depth=0): # Any, because there could be 2 copies of same list :-(
subs = kb.any(subj = c, pred = owl.disjointUnionOf);
res = [ (c, depth) ];
if subs == None:
subs = kb.each(pred = rdfs.subClassOf, obj = c);
if len(subs) > 0:
sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
for sub in subs: res += listFor(sub, depth+1)
else:
for sub in subs: res += listFor(sub, depth+1)
return res
printOrder = listFor(qu.Transaction);
for cat, depth in printOrder:
label = kb.the(subj=cat, pred=rdfs.label)
if label == None:
label = `cat`
sys.stderr.write("@@ No label for "+`cat` +"\n")
else:
label = str(label)
anchor = cat.fragid
if totals.get(cat, None) != None:
print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
numberOfMonths, indent = depth)
print "<tr><td colspan='14'> ___ </td></tr>"
print monthGridRow("Income", None, income, incomeByMonth, numberOfMonths)
print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)
print "</table>"
# Chart of income stacked up against expenses
if '--charts' in sys.argv:
print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"
writeChart(filename = "chart.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 1)
writeChart(filename = "all.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 0)
# Output totals
if (totalsFilename):
ko = kb.newFormula()
for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
ko.close()
fo = open(totalsFilename, "w")
fo.write(ko.n3String())
fo.close
if '--issues' in sys.argv:
# Generate a list of errors found
errstr = ""
for x, list in errors.items():
errstr += transactionRow(x)
for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n" # @@@ encode error string
if errstr:
print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
# List Unclassified Income and Spending
def transactionList(cat):
ts = kb.each(pred = rdf.type, obj = cat)
if len(ts) == 0: return ""
label = kb.any(cat, rdfs.label)
st = '<h2>'+label.value()+'</h2>\n'
return st + transactionTable(ts)
for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
print transactionList(cat)
print reimbursablesCheck();
internalCheck()
if 0:
print "<h2>Tax Categories</h2>"
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)
print "<h2>Tax stuff</h2>"
print "<table>"
print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
print "</table>"
# print "<h2>Personal Category total</h2>"
# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)
print
print "Note totals for tax and personal breakdowns must match."
dates = kb.statementsMatching(pred=qu.date)
print "There should be a total of %i transactions in each." % len(dates)
if 0:
print "<pre>(consistency check)"
problems = 0
for s in dates:
tra = s.subject()
types = kb.each(subj=tra, pred=rdf_type)
for typ in types:
if typ is qu.Unclassified or typ is qu.Classified:
break # ok
else:
print "@@@@ problem transcation with no classified or unclassified, with types", types
printTransactionDetails(tra)
problems = problems + 1
print problems, "problems.</pre>"
print "</body></html>"
return fatalErrors
def doCommand(serialDevice=None, outputURI=None, doTracks=1, doWaypoints=1, verbose=0):
if os.name == 'nt':
if not serialDevice: serialDevice = "com1"
phys = Win32SerialLink(serialDevice)
else:
if not serialDevice: serialDevice = "/dev/ttyS0"
# serialDevice = "/dev/cu.USA19H191P1.1"
phys = UnixSerialLink(serialDevice)
gps = Garmin(phys)
print "GPS Product ID: %d Descriptions: %s Software version: %2.2f" % \
(gps.prod_id, gps.prod_descs, gps.soft_ver)
f = formula() # Empty store of RDF data
base = uripath.base()
record = f.newBlankNode()
f.add(record, RDF.type, GPS.Record)
if doWaypoints:
# show waypoints
if verbose: print "Getting waypoints"
wpts = gps.getWaypoints()
for w in wpts:
if verbose: progress(`w`)
wpt = symbol(uripath.join(base, w.ident))
f.add(record, GPS.waypoint, wpt)
f.add(wpt, WGS.lat, obj=intern(degrees(w.slat)))
f.add(wpt, WGS.long, obj=intern(degrees(w.slon)))
if doTracks:
# show track
if verbose: print "Getting tracks"
tracks = gps.getTracks()
for t in tracks:
track = f.newBlankNode()
f.add(record, GPS.track, track)
for p in t:
if isinstance(p, TrackHdr):
if verbose: progress(`p`)
f.add(track, GPS.disp, intern(p.dspl))
f.add(track, GPS.color, intern(p.color))
f.add(track, GPS.trk_ident, intern(p.trk_ident))
else:
if verbose: progress(`p`)
point = f.newBlankNode()
f.add(track, GPS.trackpoint, point)
f.add(point, WGS.lat, obj=intern(degrees(p.slat)))
f.add(point, WGS.long, obj=intern(degrees(p.slon)))
# if verbose: progress(" time=", p.time)
# progress('p.time='+`p.time`) # @@
if p.time == 0 or p.time == 0xffffffffL:
if verbose: progress("time=%8x, ignoring" % p.time)
else:
f.add(point, WGS.time, obj=intern(isodate.fullString(TimeEpoch+p.time)))
phys.f.close() # Should really be done by the del() below, but isn't
del(phys) # close serial link (?)
f = f.close()
if verbose:
progress("Beginning output. You can disconnect the GPS now.")
s = f.n3String(base=base, flags="d") # Flag - no default prefix, preserve gps: prefix hint
if outputURI != None:
op = open(outputURI, "w")
op.write(s)
op.close()
else:
print s
def patches(delta, f, only_f, originalBnodes, definitions, deleting=0):
"""Generate patches in patch formula, for the remaining statements in f
given the bnodes and definitions for f."""
todo = only_f.copy()
if deleting:
patchVerb = DELTA.deletion
else:
patchVerb = DELTA.insertion
if verbose>2: progress("*********** PATCHES: %s, with %i to do" %(patchVerb, len(todo)))
while todo:
# find a contiguous subgraph defined in the given graph
bnodesToDo = Set()
bnodes = Set()
rhs = delta.newFormula()
lhs = delta.newFormula() # left hand side of patch
newStatements = Set()
for seed in todo: break # pick one #@2 fn?
statementsToDo = Set([seed])
if verbose>3: progress("Seed:", seed)
subgraph = statementsToDo
while statementsToDo or bnodesToDo:
for st in statementsToDo:
s, p, o = st.spo()
for x in s, p, o:
if x.generated() and x not in bnodes: # and x not in commonBnodes:
if verbose>4: progress(" Bnode ", x)
bnodesToDo.add(x)
bnodes.add(x)
rhs.add(s, p, o)
statementsToDo = Set()
for x in bnodesToDo:
bnodes.add(x)
ss = (f.statementsMatching(subj=x)
+ f.statementsMatching(pred=x)
+ f.statementsMatching(obj=x))
for z in ss:
if z in only_f:
newStatements.add(z)
if verbose>3: progress(" New statements from %s: %s" % (x, newStatements))
statementsToDo = statementsToDo | newStatements
subgraph = subgraph |newStatements
bnodesToDo = Set()
if verbose>3: progress("Subgraph of %i statements (%i left):\n\t%s\n"
%(len(subgraph), len(todo), subgraph))
todo = todo - subgraph
undefined = bnodes.copy()
for x, inverse, pred, y in definitions:
if x in undefined:
if inverse: s, p, o = x, pred, y
else: s, p, o = y, pred, x
if verbose > 4: progress("Declaring variable %s" % x.uriref())
if deleting:
delta.declareUniversal(x)
lhs.add(subj=s, pred=p, obj=o)
else: # inserting
if x in originalBnodes:
delta.declareUniversal(x)
lhs.add(subj=s, pred=p, obj=o)
else:
rhs.declareExistential(x)
if y.generated():
undefined.add(y)
undefined.discard(x)
if undefined:
progress("Still haven't defined bnodes %s" % undefined)
for n in undefined:
debugBnode(n, f)
raise RuntimeError("BNodes still undefined", undefined)
lhs = lhs.close()
rhs = rhs.close()
delta.add(subj=lhs, pred=patchVerb, obj=rhs)
if verbose >1: progress("PATCH: %s %s %s\n" %(lhs.n3String(), `patchVerb`, rhs.n3String()))
return
def main():
global verbose, proofs, chatty, normal, no_action
start = 1
cwm_command='../cwm.py'
python_command='python -tt'
global ploughOn # even if error
ploughOn = 0
global verbose
verbose = 0
global just_fix_it
just_fix_it = 0
if diag.print_all_file_names:
a = file('testfilelist','w')
a.write('')
a.close()
try:
opts, testFiles = getopt.getopt(sys.argv[1:], "h?s:nNcipf:v",
["help", "start=", "testsFrom=", "no-action", "No-normal", "chatty",
"ignoreErrors", "proofs", "verbose","overwrite","cwm="])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "-?", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose = 1
if o in ("-i", "--ignoreErrors"):
ploughOn = 1
if o in ("-s", "--start"):
start = int(a)
if o in ("-f", "--testsFrom"):
testFiles.append(a)
if o in ("-n", "--no-action"):
no_action = 1
if o in ("-N", "--No-normal"):
normal = 0
if o in ("-c", "--chatty"):
chatty = 1
if o in ("-p", "--proofs"):
proofs = 1
if o in ("--overwrite",):
just_fix_it = 1
if o in ("--cwm", "--the_end"):
cwm_command=a
assert system("mkdir -p ,temp") == 0
assert system("mkdir -p ,diffs") == 0
if proofs: assert system("mkdir -p ,proofs") == 0
tests=0
passes=0
global problems
problems = []
REFWD="http://example.com/swap/test"
WD = base()[:-1]
#def basicTest(case, desc, args)
if verbose: progress("Test files:", testFiles)
kb = loadMany(testFiles, referer="")
testData = []
RDFTestData = []
RDFNegativeTestData = []
perfData = []
n3PositiveTestData = []
n3NegativeTestData = []
sparqlTestData = []
# for fn in testFiles:
# print "Loading tests from", fn
# kb=load(fn)
for t in kb.each(pred=rdf.type, obj=test.CwmTest):
verboseDebug = kb.contains(subj=t, pred=rdf.type, obj=test.VerboseTest)
u = t.uriref()
ref = kb.the(t, test.referenceOutput)
if ref == None:
case = str(kb.the(t, test.shortFileName))
refFile = "ref/%s" % case
else:
refFile = refTo(base(), ref.uriref())
case = ""
for ch in refFile:
if ch in "/#": case += "_"
else: case += ch # Make up test-unique temp filename
description = str(kb.the(t, test.description))
arguments = str(kb.the(t, test.arguments))
environment = kb.the(t, test.environment)
if environment == None: env=""
else: env = str(environment) + " "
testData.append((t, t.uriref(), case, refFile, description, env, arguments, verboseDebug))
for t in kb.each(pred=rdf.type, obj=rdft.PositiveParserTest):
x = t.uriref()
y = x.find("/rdf-tests/")
x = x[y+11:] # rest
for i in range(len(x)):
if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
case = "rdft_" + x + ".nt" # Hack - temp file name
description = str(kb.the(t, rdft.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, rdft.inputDocument).uriref()
outputDocument = kb.the(t, rdft.outputDocument).uriref()
status = kb.the(t, rdft.status).string
good = 1
if status != "APPROVED":
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.ReificationTest:
if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
good = 0
## if cat is triage.ParseTypeLiteralTest:
## if verbose: print "\tNot supported (Parse type literal): "+ inputDocument[-40:]
## good = 0
if good:
RDFTestData.append((t.uriref(), case, description, inputDocument, outputDocument))
for t in kb.each(pred=rdf.type, obj=rdft.NegativeParserTest):
x = t.uriref()
y = x.find("/rdf-tests/")
x = x[y+11:] # rest
for i in range(len(x)):
if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
case = "rdft_" + x + ".nt" # Hack - temp file name
description = str(kb.the(t, rdft.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, rdft.inputDocument).uriref()
status = kb.the(t, rdft.status).string
good = 1
if status != "APPROVED":
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.knownError:
if verbose: print "\tknown failure: "+ inputDocument[-40:]
good = 0
if cat is triage.ReificationTest:
if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
good = 0
if good:
RDFNegativeTestData.append((t.uriref(), case, description, inputDocument))
for t in kb.each(pred=rdf.type, obj=n3test.PositiveParserTest):
u = t.uriref()
hash = u.rfind("#")
slash = u.rfind("/")
assert hash >0 and slash > 0
case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
description = str(kb.the(t, n3test.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, n3test.inputDocument).uriref()
good = 1
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.knownError:
if verbose: print "\tknown failure: "+ inputDocument[-40:]
good = 0
if good:
n3PositiveTestData.append((t.uriref(), case, description, inputDocument))
for t in kb.each(pred=rdf.type, obj=n3test.NegativeParserTest):
u = t.uriref()
hash = u.rfind("#")
slash = u.rfind("/")
assert hash >0 and slash > 0
case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
description = str(kb.the(t, n3test.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, n3test.inputDocument).uriref()
n3NegativeTestData.append((t.uriref(), case, description, inputDocument))
for tt in kb.each(pred=rdf.type, obj=sparql_manifest.Manifest):
for t in kb.the(subj=tt, pred=sparql_manifest.entries):
name = str(kb.the(subj=t, pred=sparql_manifest.name))
query_node = kb.the(subj=t, pred=sparql_manifest.action)
if isinstance(query_node, AnonymousNode):
data = ''
for data_node in kb.each(subj=query_node, pred=sparql_query.data):
data = data + ' ' + data_node.uriref()
inputDocument = kb.the(subj=query_node, pred=sparql_query.query).uriref()
else:
data = ''
inputDocument = query_node.uriref()
j = inputDocument.rfind('/')
case = inputDocument[j+1:]
outputDocument = kb.the(subj=t, pred=sparql_manifest.result)
if outputDocument:
outputDocument = outputDocument.uriref()
else:
outputDocument = None
good = 1
status = kb.the(subj=t, pred=dawg_test.approval)
if status != dawg_test.Approved:
print status, name
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
if good:
sparqlTestData.append((tt.uriref(), case, name, inputDocument, data, outputDocument))
for t in kb.each(pred=rdf.type, obj=test.PerformanceTest):
x = t.uriref()
theTime = kb.the(subj=t, pred=test.pyStones)
description = str(kb.the(t, test.description))
arguments = str(kb.the(t, test.arguments))
environment = kb.the(t, test.environment)
if environment == None: env=""
else: env = str(environment) + " "
perfData.append((x, theTime, description, env, arguments))
testData.sort()
cwmTests = len(testData)
if verbose: print "Cwm tests: %i" % cwmTests
RDFTestData.sort()
RDFNegativeTestData.sort()
rdfTests = len(RDFTestData)
rdfNegativeTests = len(RDFNegativeTestData)
perfData.sort()
perfTests = len(perfData)
n3PositiveTestData.sort()
n3PositiveTests = len(n3PositiveTestData)
n3NegativeTestData.sort()
n3NegativeTests = len(n3NegativeTestData)
sparqlTestData.sort()
sparqlTests = len(sparqlTestData)
totalTests = cwmTests + rdfTests + rdfNegativeTests + sparqlTests \
+ perfTests + n3PositiveTests + n3NegativeTests
if verbose: print "RDF parser tests: %i" % rdfTests
for t, u, case, refFile, description, env, arguments, verboseDebug in testData:
tests = tests + 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, description)
# print " %scwm %s giving %s" %(arguments, case)
assert case and description and arguments
cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e 's;%s;%s;g'""" % (WD, REFWD,
cwm_command, '../cwm.py')
if normal:
execute("""CWM_RUN_NS="run#" %s %s %s --quiet %s | %s > ,temp/%s""" %
(env, python_command, cwm_command, arguments, cleanup , case))
if diff(case, refFile):
problem("######### from normal case %s: %scwm %s" %( case, env, arguments))
continue
if chatty and not verboseDebug:
execute("""%s %s %s --chatty=100 %s &> /dev/null""" %
(env, python_command, cwm_command, arguments), noStdErr=True)
if proofs and kb.contains(subj=t, pred=rdf.type, obj=test.CwmProofTest):
execute("""%s %s %s --quiet %s --base=a --why > ,proofs/%s""" %
(env, python_command, cwm_command, arguments, case))
execute("""%s ../check.py < ,proofs/%s | %s > ,temp/%s""" %
(python_command, case, cleanup , case))
if diff(case, refFile):
problem("######### from proof case %s: %scwm %s" %( case, env, arguments))
# else:
# progress("No proof for "+`t`+ " "+`proofs`)
# progress("@@ %s" %(kb.each(t,rdf.type)))
passes = passes + 1
for u, case, name, inputDocument, data, outputDocument in sparqlTestData:
tests += 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, name)
inNtriples = case + '_1'
outNtriples = case + '_2'
try:
execute("""%s %s %s --sparql=%s --filter=%s --filter=%s --ntriples > ',temp/%s'""" %
(python_command, cwm_command, data, inputDocument,
'sparql/filter1.n3', 'sparql/filter2.n3', inNtriples))
except NotImplementedError:
pass
except:
problem(str(sys.exc_info()[1]))
if outputDocument:
execute("""%s %s %s --ntriples > ',temp/%s'""" %
(python_command, cwm_command, outputDocument, outNtriples))
if rdfcompare3(inNtriples, ',temp/' + outNtriples):
problem('We have a problem with %s on %s' % (inputDocument, data))
passes += 1
for u, case, description, inputDocument, outputDocument in RDFTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument and outputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
execute("""%s %s --quiet --rdf=RT %s --ntriples > ,temp/%s""" %
(python_command, cwm_command, inputDocument, case))
if rdfcompare3(case, localize(outputDocument)):
problem(" from positive parser test %s running\n\tcwm %s\n" %( case, inputDocument))
passes = passes + 1
for u, case, description, inputDocument in RDFNegativeTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s --quiet --rdf=RT %s --ntriples > ,temp/%s 2>/dev/null""" %
(python_command, cwm_command, inputDocument, case))
except:
pass
else:
problem("""I didn't get a parse error running python %s --quiet --rdf=RT %s --ntriples > ,temp/%s
from test ^=%s
I should have.
""" %
(cwm_command, inputDocument, case, u))
passes = passes + 1
for u, case, description, inputDocument in n3PositiveTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s 2>/dev/null""" %
(python_command, '../grammar/predictiveParser.py', inputDocument, case))
except RuntimeError:
problem("""Error running ``python %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s 2>/dev/null''""" %
('../grammar/predictiveParser.py', inputDocument, case))
passes = passes + 1
for u, case, description, inputDocument in n3NegativeTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s ../grammar/n3-selectors.n3 http://www.w3.org/2000/10/swap/grammar/n3#document %s > ,temp/%s 2>/dev/null""" %
(python_command, '../grammar/predictiveParser.py', inputDocument, case))
except:
pass
else:
problem("""There was no error executing ``python %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s''
There should have been one.""" %
('../grammar/predictiveParser.py', inputDocument, case))
passes = passes + 1
timeMatcher = re.compile(r'\t([0-9]+)m([0-9]+)\.([0-9]+)s')
## from test.pystone import pystones
## pyStoneTime = pystones()[1]
for u, theTime, description, env, arguments in perfData:
tests = tests + 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, description)
tt = os.times()[-1]
a = system("""%s %s %s --quiet %s >,time.out""" %
(env, python_command, cwm_command, arguments))
userTime = os.times()[-1] - tt
print """%spython %s --quiet %s 2>,time.out""" % \
(env, cwm_command, arguments)
## c = file(',time.out', 'r')
## timeOutput = c.read()
## c.close()
## timeList = [timeMatcher.search(b).groups() for b in timeOutput.split('\n') if timeMatcher.search(b) is not None]
## print timeList
## userTimeStr = timeList[1]
## userTime = int(userTimeStr[0])*60 + float(userTimeStr[1] + '.' + userTimeStr[2])
pyCount = pyStoneTime * userTime
print pyCount
if problems != []:
sys.stderr.write("\nProblems:\n")
for s in problems:
sys.stderr.write(" " + s + "\n")
raise RuntimeError("Total %i errors in %i tests." % (len(problems), tests))
def main():
global already, agenda, errors
parseAs = None
grammarFile = None
parseFile = None
yaccFile = None
global verbose
global g
verbose = 0
lumped = 1
try:
opts, args = getopt.getopt(
sys.argv[1:], "ha:v:p:g:y:",
["help", "as=", "verbose=", "parse=", "grammar=", "yacc="])
except getopt.GetoptError:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose = int(a)
diag.chatty_flag = int(a)
if o in ("-a", "--as"):
parseAs = uripath.join(uripath.base(), a)
if o in ("-p", "--parse"):
parseFile = uripath.join(uripath.base(), a)
if o in ("-g", "--grammar"):
grammarFile = uripath.join(uripath.base(), a)
if o in ("-y", "--yacc"):
yaccFile = uripath.join(uripath.base(), a)[5:] # strip off file:
# if testFiles == []: testFiles = [ "/dev/stdin" ]
if not parseAs:
usage()
sys.exit(2)
parseAs = uripath.join(uripath.base(), parseAs)
if not grammarFile:
grammarFile = parseAs.split("#")[0] # strip off fragid
else:
grammarFile = uripath.join(uripath.base(), grammarFile)
# The Grammar formula
progress("Loading " + grammarFile)
start = clock()
g = load(grammarFile)
taken = clock() - start + 1
progress("Loaded %i statements in %fs, ie %f/s." %
(len(g), taken, len(g) / taken))
document = g.newSymbol(parseAs)
already = []
agenda = []
errors = []
doProduction(document)
while agenda:
x = agenda[0]
agenda = agenda[1:]
already.append(x)
doProduction(x)
if errors != []:
progress("###### FAILED with %i errors." % len(errors))
for s in errors:
progress("\t%s" % s)
exit(-2)
else:
progress("Ok for predictive parsing")
#if parser.verb: progress "Branch table:", branchTable
if verbose:
progress("Literal terminals: %s" % literalTerminals.keys())
progress("Token regular expressions:")
for r in tokenRegexps:
progress("\t%s matches %s" % (r, tokenRegexps[r].pattern))
if yaccFile:
yacc = open(yaccFile, "w")
yaccConvert(yacc, document, tokenRegexps)
yacc.close()
if parseFile == None: exit(0)
ip = webAccess.urlopenForRDF(parseFile, None)
str = ip.read().decode('utf_8')
sink = g.newFormula()
keywords = g.each(pred=BNF.keywords, subj=document)
keywords = [a.value() for a in keywords]
p = PredictiveParser(sink=sink,
top=document,
branchTable=branchTable,
tokenRegexps=tokenRegexps,
keywords=keywords)
p.verb = verbose
start = clock()
p.parse(str)
taken = clock() - start + 1
progress("Loaded %i chars in %fs, ie %f/s." %
(len(str), taken, len(str) / taken))
progress("Parsed <%s> OK" % parseFile)
sys.exit(0) # didn't crash
def __init__(self, minla, maxla, minlo, maxlo, svgStream=None):
def getSize(s, atr):
i = s.find(atr + '="') + len(atr) + 2
val = ""
while s[i] in '0123456789':
val += s[i]
i = i + 1
x = int(val)
progress("Found attribute %s=%i" % (atr, x))
return x
progress("Lat between %f and %f, Long %f and %f" %
(minla, maxla, minlo, maxlo))
#pageLong_m = 10.5 * 25.4 / 1000 # Say for 8.5 x 11" US Paper YMMV
#pageShort_m = 8.0 * 25.4 / 1000 #This is printable
# Not to scale ?!!
pageLong_m = 7 * 25.4 / 1000 # Say for 8.5 x 11" US Paper YMMV
pageShort_m = 5 * 25.4 / 1000 #This is printable
if svgStream == None: self.wr = sys.stdout.write
else: self.wr = svgStream.write
self.marks = [] # List of marks on the map to avoid overlap
self.midla = (minla + maxla) / 2.0
self.midlo = (minlo + maxlo) / 2.0
self.total_m = 0 # Meters
self.last = None # (lon, lat, date)
self.t0 = isodate.parse("2035-01-01T00:00:00Z")
# Initialize to far future @@@ why doesnt 2099 or 2999 work?
self.t9 = isodate.parse("1999-01-01T00:00:00Z")
# Initialise to far past
self.speeds = []
# self.elevations = [];
pi = 3.14159265358979323846 # (say)
degree = pi / 180
# r_earth = 6400000.0 # (say) meters
phi = self.midla * degree
# See http://en.wikipedia.org/wiki/Earth_radius
a = 6.378137e6 # m
b = 6.3567523e6 #m
r_earth = sqrt(((a * a * cos(phi))**2 + ((a * a * sin(phi)))**2) /
((a * cos(phi))**2 + ((a * sin(phi)))**2))
print "Local radius of earth = ", r_earth
self.y_m_per_degree = r_earth * pi / 180
self.x_m_per_degree = self.y_m_per_degree * cos(self.midla * degree)
progress('Metres per degree: (%f,%f)' %
(self.x_m_per_degree, self.y_m_per_degree))
# OpsenStreetMap Map
hila = maxla + (maxla -
minla) * 0.1 # Make margins an extra 10% all round
hilo = maxlo + (maxlo - minlo) * 0.1
lola = minla - (maxla - minla) * 0.1
lolo = minlo - (maxlo - minlo) * 0.1
subtended_x = (hilo - lolo) * self.x_m_per_degree
subtended_y = (hila - lola) * self.y_m_per_degree
progress("Area subtended %f (E-W) %f (N-S) meters" %
(subtended_x, subtended_y))
vertical = subtended_y > subtended_x
if vertical:
if subtended_y / pageLong_m > subtended_x / pageShort_m: # constrained by height
osmScale = subtended_y / pageLong_m
hilo = self.midlo + 0.5 * (pageShort_m * osmScale /
self.x_m_per_degree)
lolo = self.midlo - 0.5 * (pageShort_m * osmScale /
self.x_m_per_degree)
else: # constrained by width
osmScale = subtended_x / pageShort_m
hila = self.midla + 0.5 * (pageLong_m * osmScale /
self.y_m_per_degree)
lola = self.midla - 0.5 * (pageLong_m * osmScale /
self.y_m_per_degree)
else:
if subtended_x / pageLong_m > subtended_y / pageShort_m: # constrained by long width
osmScale = subtended_x / pageLong_m
hila = self.midla + 0.5 * (pageShort_m * osmScale /
self.y_m_per_degree)
lola = self.midla - 0.5 * (pageShort_m * osmScale /
self.y_m_per_degree)
else: # constrained by short height
osmScale = subtended_y / pageShort_m
hilo = self.midlo + 0.5 * (pageLong_m * osmScale /
self.x_m_per_degree)
lolo = self.midlo - 0.5 * (pageLong_m * osmScale /
self.x_m_per_degree)
progress("OSM scale: %f" % osmScale)
zoom = 20 - log(osmScale / 500, 2)
progress("float zoom: %f" % zoom)
zoomLevel = int(zoom)
progress("float zoom: %i" % zoomLevel)
progress("Area subtended %f (E-W) %f (N-S) meters" %
(subtended_x, subtended_y))
pizelsPerAtZ20ATEquator = 156412.0
# osmScale = 10000 # say
#self.pixels_per_m = 122.94/25.4 * 1000 / osmScale # pixels per metre on the ground - dpi was 120 now 123
#self.pixels_per_m = 85 /25.4 * 1000 / osmScale # Calculating this doesn't sem to work -- lets look at the actual map
#self.page_x = (hilo-lolo) * self.x_m_per_degree * self.pixels_per_m
#self.page_y = (hila-lola) * self.y_m_per_degree * self.pixels_per_m
layers = 'C'
# Cyclemap
# Like http://tile.openstreetmap.org/cgi-bin/export?bbox=-71.2118,42.42694,-71.19273,42.44086&scale=25000&format=svg
# OSM_URI = ("http://tile.openstreetmap.org/cgi-bin/export?bbox=%f,%f,%f,%f&scale=%i&format=svg" % (lolo, lola, hilo, hila, osmScale))
# OSM_URI = ("http://render.openstreetmap.org/cgi-bin/export?bbox=%f,%f,%f,%f&scale=%i&format=svg" % (lolo, lola, hilo, hila, osmScale))
OSM_URI = (
"http://render.openstreetmap.org/cgi-bin/export?bbox=%f,%f,%f,%f&scale=%i&format=svg&layers=%s"
% (lolo, lola, hilo, hila, osmScale, layers))
progress("Batch OSM map at: ", OSM_URI)
interactiveMapUri = "https://openstreetmap.org/#map=%i/%f/%f&layers=C" % (
zoom, (lola + hila) / 2.0, (lolo + hilo) / 2.0)
progress("Interactive OSM map at: ", interactiveMapUri)
try:
pass
#saveAs(OSM_URI, "background-map.svg")
if false:
osmStream = urlopen(OSM_URI)
osmData = osmStream.read(
) # Beware of server overloaded errors
osmStream.close
else:
osmData = requests.get(OSM_URI,
cookies={
"_osm_totp_token": "102462"
},
headers={
'User-Agent': 'Mozilla/5.0'
}).text
except IOError:
progress("Unable to get OSM map")
sys.exit(4) # @@ should extract the error code from somwhere
i = osmData.rfind('</svg>')
if i < 0:
progress("Invalid SVG file from OSM:\n" + osmData[:1000])
sys.exit(5)
self.wr(osmData[:i]) # Everything except for the last </svg>
# Set up parametrs for point mapping:
self.page_x = getSize(osmData, 'width')
self.page_y = getSize(osmData, 'height')
self.lolo = lolo
self.lola = lola
self.hilo = hilo
self.hila = hila
self.pixels_per_deg_lat = self.page_y / (hila - lola)
self.pixels_per_deg_lon = self.page_x / (hilo - lolo)
# self.pixels_per_deg_lat = self.pixels_per_m * r_earth * pi /180
# self.pixels_per_deg_lon = self.pixels_per_deg_lat * cos(self.midla*degree)
# page_x = 800.0 # pixels
# page_y = 600.0
# max_x_scale = page_x / subtended_x
# max_y_scale = page_y / subtended_y
# self.pixels_per_m = min(max_x_scale, max_y_scale) * 0.9 # make margins
#self.page_x = int(subtended_x * self.pixels_per_m)
#self.page_y = int(subtended_y * self.pixels_per_m)
# TIGER map
if 0:
map_wid = subtended_x / 0.9
map_ht = subtended_y / 0.9
tigerURI = (
"http://tiger.census.gov/cgi-bin/mapper/map.gif?" +
"&lat=%f&lon=%f&ht=%f&wid=%f&" +
"&on=CITIES&on=majroads&on=miscell&on=places&on=railroad&on=shorelin&on=streets"
+ "&on=interstate&on=statehwy&on=states&on=ushwy&on=water" +
"&tlevel=-&tvar=-&tmeth=i&mlat=&mlon=&msym=bigdot&mlabel=&murl=&conf=mapnew.con"
+ "&iht=%i&iwd=%i") % (self.midla, self.midlo, map_ht, map_wid,
self.page_y, self.page_x)
progress("Getting tiger map ", tigerURI)
try:
saveAs(tigerURI, "tiger.gif")
except IOError:
progress("Offline? No tigermap.")
# tigerURI = ("http://tiger.census.gov/cgi-bin/mapper/map.gif?&lat=%f&lon=%f&ht=%f"
# +"&wid=%f&&on=majroads&on=miscell&tlevel=-&tvar=-&tmeth=i&mlat=&mlon=&msym=bigdot&mlabel=&murl="
# +"&conf=mapnew.con&iht=%i&iwd=%i" ) % (self.midla, self.midlo, maxla-minla, maxlo-minlo, self.page_y, self.page_x)
if 0:
self.wr("""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC '-//W3C//DTD SVG 1.0//EN'
'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd'>
<!-- Generated by @@@ -->
<svg
width="%ipx"
height="%ipx"
xmlns='http://www.w3.org/2000/svg'
xmlns:xlink='http://www.w3.org/1999/xlink'>
<g>
""" % (self.page_x, self.page_y)) #"
progress('Map page size (%f,%f)' % (self.page_x, self.page_y))
def doProduction(lhs):
"Generate branch tables for one production"
global branchTable
if lhs is BNF.void:
progress("\nvoid")
return
if lhs is BNF.eof:
progress("\nEOF")
return
if isinstance(lhs, Literal):
literalTerminals[lhs.value()] = 1
return
branchDict = {}
rhs = g.the(pred=BNF.matches, subj=lhs)
if rhs != None:
if chatty_flag: progress("\nToken %s matches regexp %s" % (lhs, rhs))
try:
tokenRegexps[lhs] = smartCompile(rhs.value(), re.U)
except:
print rhs.value().encode('utf-8')
raise
cc = g.each(subj=lhs, pred=BNF.canStartWith)
if cc == []:
progress(
recordError("No record of what token %s can start with" %
` lhs `))
if chatty_flag: progress("\tCan start with: %s" % cc)
return
if g.contains(subj=lhs, pred=RDF.type, obj=REGEX.Regex):
import regex
rhs = regex.makeRegex(g, lhs)
try:
tokenRegexps[lhs] = smartCompile(rhs, re.U)
except:
print rhs
raise
cc = g.each(subj=lhs, pred=BNF.canStartWith)
if cc == []:
progress(
recordError("No record of what token %s can start with" %
` lhs `))
if chatty_flag: progress("\tCan start with: %s" % cc)
return
rhs = g.the(pred=BNF.mustBeOneSequence, subj=lhs)
if rhs == None:
progress(recordError("No definition of " + ` lhs `))
return
# raise RuntimeError("No definition of %s in\n %s" %(`lhs`, `g`))
options = rhs
if chatty_flag:
progress("\nProduction %s :: %s ie %s" %
( ` lhs `, ` options `, ` options.value() `))
succ = g.each(subj=lhs, pred=BNF.canPrecede)
if chatty_flag: progress("\tCan precede ", succ)
branches = g.each(subj=lhs, pred=BNF.branch)
for branch in branches:
option = g.the(subj=branch, pred=BNF.sequence)
if chatty_flag: progress("\toption: " + ` option.value() `)
for part in option:
if part not in already and part not in agenda: agenda.append(part)
y = ` part `
conditions = g.each(subj=branch, pred=BNF.condition)
if conditions == []:
progress(
recordError(" NO SELECTOR for %s option %s ie %s" %
( ` lhs `, ` option `, ` option.value() `)))
if option.value == []: # Void case - the tricky one
succ = g.each(subj=lhs, pred=BNF.canPrecede)
for y in succ:
if chatty_flag: progress("\t\t\tCan precede ", ` y `)
if chatty_flag: progress("\t\tConditions: %s" % (conditions))
for str1 in conditions:
if str1 in branchDict:
progress(
recordError("Conflict: %s is also the condition for %s" %
(str1, branchDict[str1].value())))
branchDict[str1.__str__()] = option
# break
for str1 in branchDict:
for str2 in branchDict:
s1 = unicode(str1)
s2 = unicode(str2)
# @@ check that selectors are distinct, not substrings
if (s1.startswith(s2) or s2.startswith(s1)
) and branchDict[str1] is not branchDict[str2]:
progress(
"WARNING: for %s, %s indicates %s, but %s indicates %s" %
(lhs, s1, branchDict[str1], s2, branchDict[str2]))
branchTable[lhs] = branchDict
def endPath(self):
progress('Track length so far:/m: %f' % (self.total_m))
# progress('Time spent walking/min: %f' % (self.walking/60.0))
self.wr("'/>\n\n")
def doProduction(lhs):
"Generate branch tables for one production"
global branchTable
if lhs is BNF.void:
progress("\nvoid")
return
if lhs is BNF.eof:
progress("\nEOF")
return
if isinstance(lhs, Literal):
literalTerminals[lhs.value()] = 1
return
branchDict = {}
rhs = g.the(subj=lhs, obj=BNF.Token)
if rhs != None:
tokenSet.add(rhs)
return
rhs = g.the(pred=BNF.mustBeOneSequence, subj=lhs)
if rhs == None:
progress(recordError("I can't find a definition of " + ` lhs `))
return
# raise RuntimeError("No definition of %s in\n %s" %(`lhs`, `g`))
options = rhs
if chatty_flag:
progress("\nProduction %s :: %s ie %s" %
( ` lhs `, ` options `, ` options.value() `))
succ = g.each(subj=lhs, pred=BNF.canPrecede)
if chatty_flag: progress("\tCan precede ", succ)
branches = g.each(subj=lhs, pred=BNF.branch)
for branch in branches:
option = g.the(subj=branch, pred=BNF.sequence)
if chatty_flag: progress("\toption: " + ` option.value() `)
for part in option:
if part not in already and part not in agenda: agenda.append(part)
y = ` part `
conditions = g.each(subj=branch, pred=BNF.condition)
if conditions == []:
progress(
recordError(" NO SELECTOR for %s option %s ie %s" %
( ` lhs `, ` option `, ` option.value() `)))
if option.value == []: # Void case - the tricky one
succ = g.each(subj=lhs, pred=BNF.canPrecede)
for y in succ:
if chatty_flag: progress("\t\t\tCan precede ", ` y `)
if chatty_flag: progress("\t\tConditions: %s" % (conditions))
for str1 in conditions:
if str1 in branchDict:
progress(
recordError("Conflict: %s is also the condition for %s" %
(str1, branchDict[str1].value())))
branchDict[str1.__str__()] = option
# break
for str1 in branchDict:
for str2 in branchDict:
s1 = unicode(str1)
s2 = unicode(str2)
# @@ check that selectors are distinct, not substrings
if (s1.startswith(s2) or s2.startswith(s1)
) and branchDict[str1] is not branchDict[str2]:
progress(
"WARNING: for %s, %s indicates %s, but %s indicates %s" %
(lhs, s1, branchDict[str1], s2, branchDict[str2]))
branchTable[lhs] = branchDict
def main(argv):
global chatty
global debugLevelForInference
global debugLevelForParsing
global nameBlankNodes
setVerbosity(0)
policy=ParsingOK()
try:
opts, args = getopt.getopt(argv[1:], "hv:c:p:B:a",
[ "help", "verbose=", "chatty=", "parsing=", "nameBlankNodes",
"allPremises", "profile", "report"])
except getopt.GetoptError:
sys.stderr.write("check.py: Command line syntax error.\n\n")
usage()
sys.exit(2)
output = None
report = False
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
chatty = int(a)
if o in ("-p", "--verboseParsing"):
debugLevelForParsing = int(a)
if o in ("-c", "--chatty"):
debugLevelForInference = int(a)
if o in ("-B", "--nameBlankNodes"):
nameBlankNodes = 1
if o in ("-a", "--allPremises"):
policy = AllPremises()
if o in ("--profile"):
pass
if o in ("--report"):
report = True
if nameBlankNodes: flags="B"
else: flags=""
if args:
fyi("Reading proof from "+args[0])
proof = topLevelLoad(args[0], flags=flags)
else:
fyi("Reading proof from standard input.", thresh=5)
proof = topLevelLoad(flags=flags)
# setVerbosity(60)
fyi("Length of proof formula: "+`len(proof)`, thresh=5)
try:
c = Checker(proof)
if report:
sys.stdout.write(PfReportHeader)
c.report(sys.stdout)
sys.stdout.write("\n\nConclusion::\n\n")
proved = c.result(c.conjecture()[1], policy=policy)
fyi("Proof looks OK. %i Steps" % proofSteps, thresh=5)
setVerbosity(0)
txt = proved.n3String().encode('utf-8')
print "\n".join([' ' + ln.strip() for ln in txt.split("\n")])
except InvalidProof, e:
progress("Proof invalid:", e)
sys.exit(-1)
def F(s, n, m, extras):
"""
Input: a state s, and a pair of node n and m
Output: Whether adding n->m is worth persuing
"""
## extras = BindingTree()
try:
hash(n)
hash(m)
except TypeError:
return True
if n in s.map or m in s.reverseMap:
if extras is False:
progress(" -- failed because of used already")
return False
return True
if not easyMatches(s, n, m):
progress(" -- failed because of easymatches")
return False
G1 = s.problem.G1
G2 = s.problem.G2
termin1, termout1, termin2, termout2, new1, new2 = 0, 0, 0, 0, 0, 0
for obj, preds in G1.followers(n).items():
if obj in s.map:
image = s.map[obj]
e = G2.edge(m, image)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(" -- failed because of edge")
return False
if newBindings:
extras.int_or(newBindings)
else:
if obj in s.t1_in:
termin1 += 1
if obj in s.t1_out:
termout1 += 1
if obj not in s.t1_in and obj not in s.t1_out:
new1 += 1
for subj, preds in G1.predecessors(n).items():
if subj in s.map:
image = s.map[subj]
e = G2.edge(image, m)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(" -- failed because of edge")
return False
if newBindings:
extras.int_or(newBindings)
else:
if subj in s.t1_in:
termin1 += 1
if subj in s.t1_out:
termout1 += 1
if subj not in s.t1_in and subj not in s.t1_out:
new1 += 1
for obj, preds in G2.followers(m).items():
progress("checking out %s's follower %s" % (m, obj))
if obj in s.reverseMap:
image = s.reverseMap[obj]
e = G1.edge(n, image)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(" -- failed because of edge")
return False
## if newBindings:
## extras.int_or(newBindings)
else:
if obj in s.t2_in:
termin2 += 1
if obj in s.t2_out:
termout2 += 1
if obj not in s.t2_in and obj not in s.t2_out:
new2 += 1
for subj, preds in G2.predecessors(m).items():
if subj in s.reverseMap:
image = s.reverseMap[subj]
e = G1.edge(image, n)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(" -- failed because of edge")
return False
## if newBindings:
## extras.int_or(newBindings)
else:
if subj in s.t2_in:
termin2 += 1
if subj in s.t2_out:
termout2 += 1
if subj not in s.t2_in and subj not in s.t2_out:
new2 += 1
## For subgraph, change to <=
if not isoCheck(termin1, termin2, termout1, termout2, new1, new2):
progress(" -- failed because of secondary")
progress(
"termin1=%s\ntermin2=%s\ntermout1=%s\ntermout2=%s\nnew1=%s\nnew2=%s\n"
% (termin1, termin2, termout1, termout2, new1, new2)
)
return False
return hardMatches(s, n, m)
def fyi(str, level=0, thresh=50):
if chatty >= thresh:
if isinstance(str, (lambda : True).__class__):
str = str()
progress(" "*(level*4), str)
return None
def loadFiles(files):
graph = myStore.formula()
graph.setClosureMode("e") # Implement sameAs by smushing
graph = myStore.loadMany(files, openFormula=graph)
if verbose: progress("Loaded", graph, graph.__class__)
return graph
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
print __doc__
sys.exit()
if o in ("-v", "--verbose"):
verbose = 1
if o in ("-g", "--gpsData"):
gpsData = a
events = []
photoMetaFileName = commandLineArg('photometa')
if photoMetaFileName:
if verbose: progress("Loading Photo data..." + photoMetaFileName)
f = load(photoMetaFileName) # Was gpsData + "/PhotoMeta.n3"
if verbose: progress("Loaded.")
ss = f.statementsMatching(pred=FILE.date)
for s in ss:
ph = s.subject()
photo = str(ph)
date = str(s.object())
da = f.any(subj=ph, pred=EXIF.dateTime)
if da != None:
date = str(da)
else:
progress("Warning: using file date %s for %s" % (date, photo))
events.append((date, "P", (ph, photo)))
if verbose: progress("%s: %s" % (date, photo))
def F(s, n, m, extras):
"""
Input: a state s, and a pair of node n and m
Output: Whether adding n->m is worth persuing
"""
## extras = BindingTree()
try:
hash(n)
hash(m)
except TypeError:
return True
if n in s.map or m in s.reverseMap:
if extras is False:
progress(' -- failed because of used already')
return False
return True
if not easyMatches(s, n, m):
progress(' -- failed because of easymatches')
return False
G1 = s.problem.G1
G2 = s.problem.G2
termin1, termout1, termin2, termout2, new1, new2 = 0, 0, 0, 0, 0, 0
for obj, preds in G1.followers(n).items():
if obj in s.map:
image = s.map[obj]
e = G2.edge(m, image)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(' -- failed because of edge')
return False
if newBindings:
extras.int_or(newBindings)
else:
if obj in s.t1_in:
termin1 += 1
if obj in s.t1_out:
termout1 += 1
if obj not in s.t1_in and obj not in s.t1_out:
new1 += 1
for subj, preds in G1.predecessors(n).items():
if subj in s.map:
image = s.map[subj]
e = G2.edge(image, m)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(' -- failed because of edge')
return False
if newBindings:
extras.int_or(newBindings)
else:
if subj in s.t1_in:
termin1 += 1
if subj in s.t1_out:
termout1 += 1
if subj not in s.t1_in and subj not in s.t1_out:
new1 += 1
for obj, preds in G2.followers(m).items():
progress("checking out %s's follower %s" % (m, obj))
if obj in s.reverseMap:
image = s.reverseMap[obj]
e = G1.edge(n, image)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(' -- failed because of edge')
return False
## if newBindings:
## extras.int_or(newBindings)
else:
if obj in s.t2_in:
termin2 += 1
if obj in s.t2_out:
termout2 += 1
if obj not in s.t2_in and obj not in s.t2_out:
new2 += 1
for subj, preds in G2.predecessors(m).items():
if subj in s.reverseMap:
image = s.reverseMap[subj]
e = G1.edge(image, n)
newBindings = BindingTree()
if not e or not easyMatches(s, preds, e, newBindings):
progress(' -- failed because of edge')
return False
## if newBindings:
## extras.int_or(newBindings)
else:
if subj in s.t2_in:
termin2 += 1
if subj in s.t2_out:
termout2 += 1
if subj not in s.t2_in and subj not in s.t2_out:
new2 += 1
## For subgraph, change to <=
if not isoCheck(termin1, termin2, termout1, termout2, new1, new2):
progress(' -- failed because of secondary')
progress(
'termin1=%s\ntermin2=%s\ntermout1=%s\ntermout2=%s\nnew1=%s\nnew2=%s\n'
% (termin1, termin2, termout1, termout2, new1, new2))
return False
return hardMatches(s, n, m)
def doCommand():
"""Command line RDF/N3 tool
<command> <options> <steps> [--with <more args> ]
options:
--pipe Don't store, just pipe out *
steps, in order left to right:
--rdf Input & Output ** in RDF/XML insead of n3 from now on
--n3 Input & Output in N3 from now on. (Default)
--rdf=flags Input & Output ** in RDF and set given RDF flags
--n3=flags Input & Output in N3 and set N3 flags
--ntriples Input & Output in NTriples (equiv --n3=usbpartane -bySubject -quiet)
--language=x Input & Output in "x" (rdf, n3, etc) --rdf same as: --language=rdf
--languageOptions=y --n3=sp same as: --language=n3 --languageOptions=sp
--ugly Store input and regurgitate, data only, fastest *
--bySubject Store input and regurgitate in subject order *
--no No output *
(default is to store and pretty print with anonymous nodes) *
--base=<uri> Set the base URI. Input or output is done as though theis were the document URI.
--closure=flags Control automatic lookup of identifiers (see below)
<uri> Load document. URI may be relative to current directory.
--apply=foo Read rules from foo, apply to store, adding conclusions to store
--patch=foo Read patches from foo, applying insertions and deletions to store
--filter=foo Read rules from foo, apply to store, REPLACING store with conclusions
--query=foo Read a N3QL query from foo, apply it to the store, and replace the store with its conclusions
--sparql=foo Read a SPARQL query from foo, apply it to the store, and replace the store with its conclusions
--rules Apply rules in store to store, adding conclusions to store
--think as -rules but continue until no more rule matches (or forever!)
--engine=otter use otter (in your $PATH) instead of llyn for linking, etc
--why Replace the store with an explanation of its contents
--why=u proof tries to be shorter
--mode=flags Set modus operandi for inference (see below)
--reify Replace the statements in the store with statements describing them.
--dereify Undo the effects of --reify
--flatten Reify only nested subexpressions (not top level) so that no {} remain.
--unflatten Undo the effects of --flatten
--think=foo as -apply=foo but continue until no more rule matches (or forever!)
--purge Remove from store any triple involving anything in class log:Chaff
--data Remove all except plain RDF triples (formulae, forAll, etc)
--strings Dump :s to stdout ordered by :k whereever { :k log:outputString :s }
--crypto Enable processing of crypto builtin functions. Requires python crypto.
--help print this message
--revision print CVS revision numbers of major modules
--chatty=50 Verbose debugging output of questionable use, range 0-99
--sparqlServer instead of outputting, start a SPARQL server on port 8000 of the store
--sparqlResults After sparql query, print in sparqlResults format instead of rdf
finally:
--with Pass any further arguments to the N3 store as os:argv values
* mutually exclusive
** doesn't work for complex cases :-/
Examples:
cwm --rdf foo.rdf --n3 --pipe Convert from rdf/xml to rdf/n3
cwm foo.n3 bar.n3 --think Combine data and find all deductions
cwm foo.n3 --flat --n3=spart
Mode flags affect inference extedning to the web:
r Needed to enable any remote stuff.
a When reading schema, also load rules pointed to by schema (requires r, s)
E Errors loading schemas of definitive documents are ignored
m Schemas and definitive documents laoded are merged into the meta knowledge
(otherwise they are consulted independently)
s Read the schema for any predicate in a query.
u Generate unique ids using a run-specific
Closure flags are set to cause the working formula to be automatically exapnded to
the closure under the operation of looking up:
s the subject of a statement added
p the predicate of a statement added
o the object of a statement added
t the object of an rdf:type statement added
i any owl:imports documents
r any doc:rules documents
E errors are ignored --- This is independant of --mode=E
n Normalize IRIs to URIs
e Smush together any nodes which are = (owl:sameAs)
See http://www.w3.org/2000/10/swap/doc/cwm for more documentation.
Setting the environment variable CWM_RDFLIB to 1 maked Cwm use rdflib to parse
rdf/xml files. Note that this requires rdflib.
"""
import time
import sys
from swap import myStore
# These would just be attributes if this were an object
global _store
global workingContext
option_need_rdf_sometime = 0 # If we don't need it, don't import it
# (to save errors where parsers don't exist)
option_pipe = 0 # Don't store, just pipe though
option_inputs = []
option_reify = 0 # Flag: reify on output (process?)
option_flat = 0 # Flag: reify on output (process?)
option_crypto = 0 # Flag: make cryptographic algorithms available
setTracking(0)
option_outURI = None
option_outputStyle = "-best"
_gotInput = 0 # Do we not need to take input from stdin?
option_meta = 0
option_normalize_iri = 0
option_flags = { "rdf":"l", "n3":"", "think":"", "sparql":""}
# RDF/XML serializer can't do list ("collection") syntax.
option_quiet = 0
option_with = None # Command line arguments made available to N3 processing
option_engine = "llyn"
option_why = ""
_step = 0 # Step number used for metadata
_genid = 0
hostname = "localhost" # @@@@@@@@@@@ Get real one
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
option_format = "n3" # set the default format
option_first_format = None
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# First pass on command line - - - - - - - P A S S 1
for argnum in range(1,len(sys.argv)): # options after script name
arg = sys.argv[argnum]
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
# _equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
try:
[_lhs,_rhs]=arg.split('=',1)
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
except ValueError: pass
if arg == "-ugly": option_outputStyle = arg
elif _lhs == "-base": option_baseURI = _uri
elif arg == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_need_rdf_sometime = 1
elif _lhs == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_flags["rdf"] = _rhs
option_need_rdf_sometime = 1
elif arg == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
option_flags["n3"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
if "n" in _rhs:
option_normalize_iri = 1
#elif _lhs == "-solve":
# sys.argv[argnum+1:argnum+1] = ['-think', '-filter=' + _rhs]
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _rhs
elif arg == "-quiet": option_quiet = 1
elif arg == "-pipe": option_pipe = 1
elif arg == "-crypto": option_crypto = 1
elif _lhs == "-why":
diag.tracking=1
diag.setTracking(1)
option_why = _rhs
elif arg == "-why":
diag.tracking=1
diag.setTracking(1)
option_why = ""
elif arg == "-track":
diag.tracking=1
diag.setTracking(1)
elif arg == "-bySubject": option_outputStyle = arg
elif arg == "-no": option_outputStyle = "-no"
elif arg == "-debugString": option_outputStyle = "-debugString"
elif arg == "-strings": option_outputStyle = "-no"
elif arg == "-sparqlResults": option_outputStyle = "-no"
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "bravestpun"
option_outputStyle = "-bySubject"
option_quiet = 1
elif _lhs == "-outURI": option_outURI = _uri
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-apply=": pass
elif arg[:7] == "-patch=": pass
elif arg == "-reify": option_reify = 1
elif arg == "-flat": option_flat = 1
elif arg == "-help":
print doCommand.__doc__
print notation3.ToN3.flagDocumentation
print toXML.ToRDF.flagDocumentation
try:
from swap import sax2rdf # RDF1.0 syntax parser to N3 RDF stream
print sax2rdf.RDFXMLParser.flagDocumentation
except:
pass
return
elif arg == "-revision":
progress( "cwm=",cvsRevision, "llyn=", llyn.cvsRevision)
return
elif arg == "-with":
option_with = sys.argv[argnum+1:] # The rest of the args are passed to n3
break
elif arg[0] == "-": pass # Other option
else :
option_inputs.append(join(option_baseURI, arg))
_gotInput = _gotInput + 1 # input filename
# Between passes, prepare for processing
setVerbosity(0)
if not option_normalize_iri:
llyn.canonical = lambda x: x
# Base defauts
if option_baseURI == _baseURI: # Base not specified explicitly - special case
if _outURI == _baseURI: # Output name not specified either
if _gotInput == 1: # But input file *is*,
_outURI = option_inputs[0] # Just output to same URI
option_baseURI = _outURI # using that as base.
if diag.tracking:
_outURI = RDFSink.runNamespace()[:-1]
option_baseURI = _outURI
option_baseURI = splitFrag(option_baseURI)[0]
# Fix the output sink
if option_format == "rdf":
_outSink = toXML.ToRDF(sys.stdout, _outURI, base=option_baseURI, flags=option_flags["rdf"])
elif option_format == "n3" or option_format == "sparql":
_outSink = notation3.ToN3(sys.stdout.write, base=option_baseURI,
quiet=option_quiet, flags=option_flags["n3"])
elif option_format == "trace":
_outSink = RDFSink.TracingRDFSink(_outURI, base=option_baseURI,
flags=option_flags.get("trace",""))
if option_pipe:
# this is really what a parser wants to dump to
_outSink.backing = llyn.RDFStore( _outURI+"#_g",
argv=option_with, crypto=option_crypto)
else:
# this is really what a store wants to dump to
_outSink.backing = notation3.ToN3(sys.stdout.write,
base=option_baseURI, quiet=option_quiet,
flags=option_flags["n3"])
# hm. why does TimBL use sys.stdout.write, above? performance at the
else:
raise NotImplementedError
version = "$Id: cwm.py,v 1.197 2007/12/13 15:38:39 syosi Exp $"
if not option_quiet and option_outputStyle != "-no":
_outSink.makeComment("Processed by " + version[1:-1]) # Strip $ to disarm
_outSink.makeComment(" using base " + option_baseURI)
if option_flat:
_outSink = notation3.Reifier(_outSink, _outURI+ "#_formula", flat=1)
if diag.tracking:
myReason = BecauseOfCommandLine(`sys.argv`)
# @@ add user, host, pid, pwd, date time? Privacy!
else:
myReason = None
if option_pipe:
_store = _outSink
workingContext = _outSink #.newFormula()
else:
if "u" in option_flags["think"]:
_store = llyn.RDFStore(argv=option_with, crypto=option_crypto)
else:
_store = llyn.RDFStore( _outURI+"#_g",
argv=option_with, crypto=option_crypto)
myStore.setStore(_store)
if _gotInput:
workingContext = _store.newFormula(option_inputs [0]+"#_work")
newTopLevelFormula(workingContext)
else: # default input
if option_first_format is None: option_first_format = option_format
ContentType={ "rdf": "application/xml+rdf", "n3":
"text/rdf+n3", "sparql":
"x-application/sparql"}[option_first_format]
workingContext = _store.load(
# asIfFrom = join(_baseURI, ".stdin"),
asIfFrom = _baseURI,
contentType = ContentType,
flags = option_flags[option_first_format],
remember = 0,
referer = "",
why = myReason, topLevel=True)
workingContext.reopen()
workingContext.stayOpen = 1 # Never canonicalize this. Never share it.
# ____________________________________________________________________
# Take commands from command line:- - - - - P A S S 2
option_format = "n3" # Use RDF/n3 rather than RDF/XML
option_flags = { "rdf":"l", "n3":"", "think": "", "sparql":"" }
option_quiet = 0
_outURI = _baseURI
option_baseURI = _baseURI # To start with
def filterize():
"""implementation of --filter
for the --filter command, so we don't have it printed twice
"""
global workingContext
global r
workingContext = workingContext.canonicalize()
_store._formulaeOfLength = {}
filterContext = _store.newFormula()
newTopLevelFormula(filterContext)
_store.load(_uri, openFormula=filterContext,
why=myReason, referer="")
_newContext = _store.newFormula()
newTopLevelFormula(_newContext)
applyRules(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
sparql_query_formula = None
for arg in sys.argv[1:]: # Command line options after script name
if verbosity()>5: progress("Processing %s." % (arg))
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
_equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
if _equals >=0:
_lhs = arg[:_equals]
_rhs = arg[_equals+1:]
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri =_rhs
if arg[0] != "-":
_inputURI = join(option_baseURI, splitFrag(arg)[0])
assert ':' in _inputURI
ContentType={ "rdf": "application/xml+rdf", "n3":
"text/rdf+n3",
"sparql": "x-application/sparql"}[option_format]
if not option_pipe: workingContext.reopen()
try:
load(_store, _inputURI,
openFormula=workingContext,
contentType =ContentType,
flags=option_flags[option_format],
referer="",
why=myReason)
except:
progress(_inputURI)
raise
_gotInput = 1
elif arg == "-help":
pass # shouldn't happen
elif arg == "-revision":
pass
elif _lhs == "-base":
option_baseURI = _uri
if verbosity() > 10: progress("Base now "+option_baseURI)
elif arg == "-ugly":
option_outputStyle = arg
elif arg == "-crypto": pass
elif arg == "-pipe": pass
elif _lhs == "-outURI": option_outURI = _uri
elif arg == "-rdf": option_format = "rdf"
elif _lhs == "-rdf":
option_format = "rdf"
option_flags["rdf"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
workingContext.setClosureMode(_rhs)
elif arg == "-n3": option_format = "n3"
elif _lhs == "-n3":
option_format = "n3"
option_flags["n3"] = _rhs
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _lhs
elif arg == "-quiet" : option_quiet = 1
elif _lhs == "-chatty": setVerbosity(int(_rhs))
elif arg[:7] == "-track=":
diag.tracking = int(_rhs)
elif option_pipe: ############## End of pipable options
print "# Command line error: %s illegal option with -pipe", arg
break
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "spartan"
option_outputStyle = "-bySubject"
option_quiet = 1
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-debugString":
option_outputStyle = arg
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext);
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext);
elif arg[:7] == "-patch=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
patch(workingContext, filterContext);
elif _lhs == "-filter":
filterize()
elif _lhs == "-query":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
_newContext = _store.newFormula()
applyQueries(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
elif _lhs == "-sparql":
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri, why=myReason,
referer="", contentType="x-application/sparql")
_newContext = _store.newFormula()
_newContext.stayOpen = True
sparql_query_formula = filterContext
applySparqlQueries(workingContext, filterContext, _newContext)
# workingContext.close()
workingContext = _newContext
elif _lhs == "-why" or arg == "-why":
workingContext.stayOpen = False
workingContext = workingContext.close()
workingContext = explainFormula(workingContext, option_why)
# Can't prove proofs
diag.tracking=0
diag.setTracking(0)
elif arg == "-dump":
workingContext = workingContext.canonicalize()
progress("\nDump of working formula:\n" + workingContext.debugString())
elif arg == "-purge":
workingContext.reopen()
_store.purge(workingContext)
elif arg == "-purge-rules" or arg == "-data":
workingContext.reopen()
_store.purgeExceptData(workingContext)
elif arg == "-rules":
workingContext.reopen()
applyRules(workingContext, workingContext)
elif arg[:7] == "-think=":
filterContext = _store.load(_uri, referer="", why=myReason, topLevel=True)
if verbosity() > 4:
progress( "Input rules to --think from " + _uri)
workingContext.reopen()
think(workingContext, filterContext, mode=option_flags["think"])
elif arg[:7] == "-solve=":
# --solve is a combination of --think and --filter.
think(workingContext, mode=option_flags["think"])
filterize()
elif _lhs == "-engine":
option_engine = _rhs
elif arg == "-think":
workingContext.isWorkingContext = True
think(workingContext, mode=option_flags["think"])
elif arg == '-rete':
from swap import pycwmko
pythink = pycwmko.directPychinkoQuery(workingContext)
#return
#pythink()
"""
from pychinko import interpreter
from swap.set_importer import Set, ImmutableSet
pyf = pycwmko.N3Loader.N3Loader()
conv = pycwmko.ToPyStore(pyf)
conv.statements(workingContext)
interp = interpreter.Interpreter(pyf.rules[:])
interp.addFacts(Set(pyf.facts), initialSet=True)
interp.run()
pyf.facts = interp.totalFacts
workingContext = workingContext.store.newFormula()
reconv = pycwmko.FromPyStore(workingContext, pyf)
reconv.run()
"""
elif arg == '-sparqlServer':
from swap.sparql import webserver
from swap import cwm_sparql
sandBoxed(True)
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
def _handler(s):
return cwm_sparql.sparql_queryString(workingContext, s)
webserver.sparql_handler = _handler
webserver.run()
elif arg == "-lxkbdump": # just for debugging
raise NotImplementedError
elif arg == "-lxfdump": # just for debugging
raise NotImplementedError
elif _lhs == "-prove":
# code copied from -filter without really being understood -sdh
_tmpstore = llyn.RDFStore( _outURI+"#_g", metaURI=_metaURI, argv=option_with, crypto=option_crypto)
tmpContext = _tmpstore.newFormula(_uri+ "#_formula")
_newURI = join(_baseURI, "_w_"+`_genid`) # Intermediate
_genid = _genid + 1
_newContext = _tmpstore.newFormula(_newURI+ "#_formula")
_tmpstore.loadURI(_uri)
print targetkb
elif arg == "-flatten":
#raise NotImplementedError
from swap import reify
workingContext = reify.flatten(workingContext)
elif arg == "-unflatten":
from swap import reify
workingContext = reify.unflatten(workingContext)
#raise NotImplementedError
elif arg == "-reify":
from swap import reify
workingContext = reify.reify(workingContext)
elif arg == "-dereify":
from swap import reify
workingContext = reify.dereify(workingContext)
elif arg == "-size":
progress("Size: %i statements in store, %i in working formula."
%(_store.size, workingContext.size()))
elif arg == "-strings": # suppress output
workingContext.outputStrings()
option_outputStyle = "-no"
elif arg == '-sparqlResults':
from cwm_sparql import outputString, SPARQL_NS
ns = _store.newSymbol(SPARQL_NS)
if not sparql_query_formula:
raise ValueError('No query')
else:
sys.stdout.write(outputString(sparql_query_formula, workingContext)[0].encode('utf_8'))
option_outputStyle = "-no"
elif arg == "-no": # suppress output
option_outputStyle = arg
elif arg[:8] == "-outURI=": pass
elif arg == "-with": break
else:
progress( "cwm: Unknown option: " + arg)
sys.exit(-1)
# Squirt it out if not piped
workingContext.stayOpen = 0 # End its use as an always-open knoweldge base
if option_pipe:
workingContext.endDoc()
else:
if hasattr(_outSink, "serializeKB"):
raise NotImplementedError
else:
if verbosity()>5: progress("Begining output.")
workingContext = workingContext.close()
assert workingContext.canonical != None
if option_outputStyle == "-ugly":
_store.dumpChronological(workingContext, _outSink)
elif option_outputStyle == "-bySubject":
_store.dumpBySubject(workingContext, _outSink)
elif option_outputStyle == "-no":
pass
elif option_outputStyle == "-debugString":
print workingContext.debugString()
else: # "-best"
_store.dumpNested(workingContext, _outSink,
flags=option_flags[option_format])
def debugBnode(n, f):
progress("For node %s" % `n`)
for s in f.statementsMatching(subj=n):
progress(" %s %s; # could be ifp?" %(`s.predicate()`, `s.object()`))
for s in f.statementsMatching(obj=n):
progress(" is %s of %s; # could be fp?" %(`s.predicate()`, `s.subject()`))
def yaccProduction(yacc, lhs, tokenRegexps):
if lhs is BNF.void:
if chatty_flag: progress( "\nvoid")
return
if lhs is BNF.eof:
if chatty_flag: progress( "\nEOF")
return
if isinstance(lhs, Literal):
literalTerminals[lhs.value()] = 1
# print "\nLiteral %s" %(lhs)
return
rhs = g.the(pred=BNF.matches, subj=lhs)
if rhs != None:
if chatty_flag: progress( "\nToken %s matches regexp %s" %(lhs, rhs))
# tokenRegexps[lhs] = re.compile(rhs.value())
return
rhs = g.the(pred=BNF.mustBeOneSequence, subj=lhs)
if rhs == None:
progress( recordError("No definition of " + `lhs`))
raise ValueError("No definition of %s in\n %s" %(`lhs`, `g`))
options = rhs
if chatty_flag:
progress ("\nProduction %s :: %s ie %s" %(`lhs`, `options` , `options.value()`))
yacc.write("\n%s:" % toYacc(lhs, tokenRegexps))
branches = g.each(subj=lhs, pred=BNF.branch)
first = 1
for branch in branches:
if not first:
yacc.write("\t|\t")
first = 0
option = g.the(subj=branch, pred=BNF.sequence)
if chatty_flag: progress( "\toption: "+`option.value()`)
yacc.write("\t")
if option.value() == [] and yacc: yacc.write(" /* empty */")
for part in option:
if part not in already and part not in agenda: agenda.append(part)
yacc.write(" %s" % toYacc(part, tokenRegexps))
yacc.write("\n")
yacc.write("\t;\n")
def doCommand():
"""Command line RDF/N3 tool
<command> <options> <steps> [--with <more args> ]
options:
--pipe Don't store, just pipe out *
steps, in order left to right:
--rdf Input & Output ** in RDF/XML insead of n3 from now on
--n3 Input & Output in N3 from now on. (Default)
--rdf=flags Input & Output ** in RDF and set given RDF flags
--n3=flags Input & Output in N3 and set N3 flags
--ntriples Input & Output in NTriples (equiv --n3=usbpartane -bySubject -quiet)
--language=x Input & Output in "x" (rdf, n3, etc) --rdf same as: --language=rdf
--languageOptions=y --n3=sp same as: --language=n3 --languageOptions=sp
--ugly Store input and regurgitate, data only, fastest *
--bySubject Store input and regurgitate in subject order *
--no No output *
(default is to store and pretty print with anonymous nodes) *
--base=<uri> Set the base URI. Input or output is done as though theis were the document URI.
--closure=flags Control automatic lookup of identifiers (see below)
<uri> Load document. URI may be relative to current directory.
--apply=foo Read rules from foo, apply to store, adding conclusions to store
--patch=foo Read patches from foo, applying insertions and deletions to store
--filter=foo Read rules from foo, apply to store, REPLACING store with conclusions
--query=foo Read a N3QL query from foo, apply it to the store, and replace the store with its conclusions
--sparql=foo Read a SPARQL query from foo, apply it to the store, and replace the store with its conclusions
--rules Apply rules in store to store, adding conclusions to store
--think as -rules but continue until no more rule matches (or forever!)
--engine=otter use otter (in your $PATH) instead of llyn for linking, etc
--why Replace the store with an explanation of its contents
--why=u proof tries to be shorter
--mode=flags Set modus operandi for inference (see below)
--reify Replace the statements in the store with statements describing them.
--dereify Undo the effects of --reify
--flatten Reify only nested subexpressions (not top level) so that no {} remain.
--unflatten Undo the effects of --flatten
--think=foo as -apply=foo but continue until no more rule matches (or forever!)
--purge Remove from store any triple involving anything in class log:Chaff
--data Remove all except plain RDF triples (formulae, forAll, etc)
--strings Dump :s to stdout ordered by :k whereever { :k log:outputString :s }
--crypto Enable processing of crypto builtin functions. Requires python crypto.
--help print this message
--revision print CVS revision numbers of major modules
--chatty=50 Verbose debugging output of questionable use, range 0-99
--sparqlServer instead of outputting, start a SPARQL server on port 8000 of the store
--sparqlResults After sparql query, print in sparqlResults format instead of rdf
finally:
--with Pass any further arguments to the N3 store as os:argv values
* mutually exclusive
** doesn't work for complex cases :-/
Examples:
cwm --rdf foo.rdf --n3 --pipe Convert from rdf/xml to rdf/n3
cwm foo.n3 bar.n3 --think Combine data and find all deductions
cwm foo.n3 --flat --n3=spart
Mode flags affect inference extedning to the web:
r Needed to enable any remote stuff.
a When reading schema, also load rules pointed to by schema (requires r, s)
E Errors loading schemas of definitive documents are ignored
m Schemas and definitive documents laoded are merged into the meta knowledge
(otherwise they are consulted independently)
s Read the schema for any predicate in a query.
u Generate unique ids using a run-specific
Closure flags are set to cause the working formula to be automatically exapnded to
the closure under the operation of looking up:
s the subject of a statement added
p the predicate of a statement added
o the object of a statement added
t the object of an rdf:type statement added
i any owl:imports documents
r any doc:rules documents
E errors are ignored --- This is independant of --mode=E
n Normalize IRIs to URIs
e Smush together any nodes which are = (owl:sameAs)
See http://www.w3.org/2000/10/swap/doc/cwm for more documentation.
Setting the environment variable CWM_RDFLIB to 1 maked Cwm use rdflib to parse
rdf/xml files. Note that this requires rdflib.
"""
import time
import sys
from swap import myStore
# These would just be attributes if this were an object
global _store
global workingContext
option_need_rdf_sometime = 0 # If we don't need it, don't import it
# (to save errors where parsers don't exist)
option_pipe = 0 # Don't store, just pipe though
option_inputs = []
option_reify = 0 # Flag: reify on output (process?)
option_flat = 0 # Flag: reify on output (process?)
option_crypto = 0 # Flag: make cryptographic algorithms available
setTracking(0)
option_outURI = None
option_outputStyle = "-best"
_gotInput = 0 # Do we not need to take input from stdin?
option_meta = 0
option_normalize_iri = 0
option_flags = {"rdf": "l", "n3": "", "think": "", "sparql": ""}
# RDF/XML serializer can't do list ("collection") syntax.
option_quiet = 0
option_with = None # Command line arguments made available to N3 processing
option_engine = "llyn"
option_why = ""
_step = 0 # Step number used for metadata
_genid = 0
hostname = "localhost" # @@@@@@@@@@@ Get real one
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
option_format = "n3" # set the default format
option_first_format = None
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# First pass on command line - - - - - - - P A S S 1
for argnum in range(1, len(sys.argv)): # options after script name
arg = sys.argv[argnum]
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
# _equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
try:
[_lhs, _rhs] = arg.split('=', 1)
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
except ValueError:
pass
if arg == "-ugly": option_outputStyle = arg
elif _lhs == "-base": option_baseURI = _uri
elif arg == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_need_rdf_sometime = 1
elif _lhs == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_flags["rdf"] = _rhs
option_need_rdf_sometime = 1
elif arg == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
option_flags["n3"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
if "n" in _rhs:
option_normalize_iri = 1
#elif _lhs == "-solve":
# sys.argv[argnum+1:argnum+1] = ['-think', '-filter=' + _rhs]
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _rhs
elif arg == "-quiet":
option_quiet = 1
elif arg == "-pipe":
option_pipe = 1
elif arg == "-crypto":
option_crypto = 1
elif _lhs == "-why":
diag.tracking = 1
diag.setTracking(1)
option_why = _rhs
elif arg == "-why":
diag.tracking = 1
diag.setTracking(1)
option_why = ""
elif arg == "-track":
diag.tracking = 1
diag.setTracking(1)
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-no":
option_outputStyle = "-no"
elif arg == "-debugString":
option_outputStyle = "-debugString"
elif arg == "-strings":
option_outputStyle = "-no"
elif arg == "-sparqlResults":
option_outputStyle = "-no"
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "bravestpun"
option_outputStyle = "-bySubject"
option_quiet = 1
elif _lhs == "-outURI":
option_outURI = _uri
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-apply=":
pass
elif arg[:7] == "-patch=":
pass
elif arg == "-reify":
option_reify = 1
elif arg == "-flat":
option_flat = 1
elif arg == "-help":
print doCommand.__doc__
print notation3.ToN3.flagDocumentation
print toXML.ToRDF.flagDocumentation
try:
from swap import sax2rdf # RDF1.0 syntax parser to N3 RDF stream
print sax2rdf.RDFXMLParser.flagDocumentation
except:
pass
return
elif arg == "-revision":
progress("cwm=", cvsRevision, "llyn=", llyn.cvsRevision)
return
elif arg == "-with":
option_with = sys.argv[argnum +
1:] # The rest of the args are passed to n3
break
elif arg[0] == "-":
pass # Other option
else:
option_inputs.append(join(option_baseURI, arg))
_gotInput = _gotInput + 1 # input filename
# Between passes, prepare for processing
setVerbosity(0)
if not option_normalize_iri:
llyn.canonical = lambda x: x
# Base defauts
if option_baseURI == _baseURI: # Base not specified explicitly - special case
if _outURI == _baseURI: # Output name not specified either
if _gotInput == 1: # But input file *is*,
_outURI = option_inputs[0] # Just output to same URI
option_baseURI = _outURI # using that as base.
if diag.tracking:
_outURI = RDFSink.runNamespace()[:-1]
option_baseURI = _outURI
option_baseURI = splitFrag(option_baseURI)[0]
# Fix the output sink
if option_format == "rdf":
_outSink = toXML.ToRDF(sys.stdout,
_outURI,
base=option_baseURI,
flags=option_flags["rdf"])
elif option_format == "n3" or option_format == "sparql":
_outSink = notation3.ToN3(sys.stdout.write,
base=option_baseURI,
quiet=option_quiet,
flags=option_flags["n3"])
elif option_format == "trace":
_outSink = RDFSink.TracingRDFSink(_outURI,
base=option_baseURI,
flags=option_flags.get("trace", ""))
if option_pipe:
# this is really what a parser wants to dump to
_outSink.backing = llyn.RDFStore(_outURI + "#_g",
argv=option_with,
crypto=option_crypto)
else:
# this is really what a store wants to dump to
_outSink.backing = notation3.ToN3(sys.stdout.write,
base=option_baseURI,
quiet=option_quiet,
flags=option_flags["n3"])
# hm. why does TimBL use sys.stdout.write, above? performance at the
else:
raise NotImplementedError
version = "$Id$"
if not option_quiet and option_outputStyle != "-no":
_outSink.makeComment("Processed by " +
version[1:-1]) # Strip $ to disarm
_outSink.makeComment(" using base " + option_baseURI)
if option_flat:
_outSink = notation3.Reifier(_outSink, _outURI + "#_formula", flat=1)
if diag.tracking:
myReason = BecauseOfCommandLine( ` sys.argv `)
# @@ add user, host, pid, pwd, date time? Privacy!
else:
myReason = None
if option_pipe:
_store = _outSink
workingContext = _outSink #.newFormula()
else:
if "u" in option_flags["think"]:
_store = llyn.RDFStore(argv=option_with, crypto=option_crypto)
else:
_store = llyn.RDFStore(_outURI + "#_g",
argv=option_with,
crypto=option_crypto)
myStore.setStore(_store)
if _gotInput:
workingContext = _store.newFormula(option_inputs[0] + "#_work")
newTopLevelFormula(workingContext)
else: # default input
if option_first_format is None: option_first_format = option_format
ContentType = {
"rdf": "application/xml+rdf",
"n3": "text/n3",
"sparql": "x-application/sparql"
}[option_first_format]
workingContext = _store.load(
# asIfFrom = join(_baseURI, ".stdin"),
asIfFrom=_baseURI,
contentType=ContentType,
flags=option_flags[option_first_format],
remember=0,
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
workingContext.stayOpen = 1 # Never canonicalize this. Never share it.
# ____________________________________________________________________
# Take commands from command line:- - - - - P A S S 2
option_format = "n3" # Use RDF/n3 rather than RDF/XML
option_flags = {"rdf": "l", "n3": "", "think": "", "sparql": ""}
option_quiet = 0
_outURI = _baseURI
option_baseURI = _baseURI # To start with
def filterize():
"""implementation of --filter
for the --filter command, so we don't have it printed twice
"""
global workingContext
global r
workingContext = workingContext.canonicalize()
_store._formulaeOfLength = {}
filterContext = _store.newFormula()
newTopLevelFormula(filterContext)
_store.load(_uri, openFormula=filterContext, why=myReason, referer="")
_newContext = _store.newFormula()
newTopLevelFormula(_newContext)
applyRules(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
sparql_query_formula = None
for arg in sys.argv[1:]: # Command line options after script name
if verbosity() > 5: progress("Processing %s." % (arg))
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
_equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
if _equals >= 0:
_lhs = arg[:_equals]
_rhs = arg[_equals + 1:]
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
if arg[0] != "-":
_inputURI = join(option_baseURI, splitFrag(arg)[0])
assert ':' in _inputURI
ContentType = {
"rdf": "application/xml+rdf",
"n3": "text/n3",
"sparql": "x-application/sparql"
}[option_format]
if not option_pipe: workingContext.reopen()
try:
load(_store,
_inputURI,
openFormula=workingContext,
contentType=ContentType,
flags=option_flags[option_format],
referer="",
why=myReason)
except:
progress(_inputURI)
raise
_gotInput = 1
elif arg == "-help":
pass # shouldn't happen
elif arg == "-revision":
pass
elif _lhs == "-base":
option_baseURI = _uri
if verbosity() > 10: progress("Base now " + option_baseURI)
elif arg == "-ugly":
option_outputStyle = arg
elif arg == "-crypto":
pass
elif arg == "-pipe":
pass
elif _lhs == "-outURI":
option_outURI = _uri
elif arg == "-rdf":
option_format = "rdf"
elif _lhs == "-rdf":
option_format = "rdf"
option_flags["rdf"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
workingContext.setClosureMode(_rhs)
elif arg == "-n3":
option_format = "n3"
elif _lhs == "-n3":
option_format = "n3"
option_flags["n3"] = _rhs
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _lhs
elif arg == "-quiet":
option_quiet = 1
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-track=":
diag.tracking = int(_rhs)
elif option_pipe: ############## End of pipable options
print "# Command line error: %s illegal option with -pipe", arg
break
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "spartan"
option_outputStyle = "-bySubject"
option_quiet = 1
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-debugString":
option_outputStyle = arg
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext)
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext)
elif arg[:7] == "-patch=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
patch(workingContext, filterContext)
elif _lhs == "-filter":
filterize()
elif _lhs == "-query":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
_newContext = _store.newFormula()
applyQueries(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
elif _lhs == "-sparql":
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
why=myReason,
referer="",
contentType="x-application/sparql")
_newContext = _store.newFormula()
_newContext.stayOpen = True
sparql_query_formula = filterContext
applySparqlQueries(workingContext, filterContext, _newContext)
# workingContext.close()
workingContext = _newContext
elif _lhs == "-why" or arg == "-why":
workingContext.stayOpen = False
workingContext = workingContext.close()
workingContext = explainFormula(workingContext, option_why)
# Can't prove proofs
diag.tracking = 0
diag.setTracking(0)
elif arg == "-dump":
workingContext = workingContext.canonicalize()
progress("\nDump of working formula:\n" +
workingContext.debugString())
elif arg == "-purge":
workingContext.reopen()
_store.purge(workingContext)
elif arg == "-purge-rules" or arg == "-data":
workingContext.reopen()
_store.purgeExceptData(workingContext)
elif arg == "-rules":
workingContext.reopen()
applyRules(workingContext, workingContext)
elif arg[:7] == "-think=":
filterContext = _store.load(_uri,
referer="",
why=myReason,
topLevel=True)
if verbosity() > 4:
progress("Input rules to --think from " + _uri)
workingContext.reopen()
think(workingContext, filterContext, mode=option_flags["think"])
elif arg[:7] == "-solve=":
# --solve is a combination of --think and --filter.
think(workingContext, mode=option_flags["think"])
filterize()
elif _lhs == "-engine":
option_engine = _rhs
elif arg == "-think":
workingContext.isWorkingContext = True
think(workingContext, mode=option_flags["think"])
elif arg == '-rete':
from swap import pycwmko
pythink = pycwmko.directPychinkoQuery(workingContext)
#return
#pythink()
"""
from pychinko import interpreter
from swap.set_importer import Set, ImmutableSet
pyf = pycwmko.N3Loader.N3Loader()
conv = pycwmko.ToPyStore(pyf)
conv.statements(workingContext)
interp = interpreter.Interpreter(pyf.rules[:])
interp.addFacts(Set(pyf.facts), initialSet=True)
interp.run()
pyf.facts = interp.totalFacts
workingContext = workingContext.store.newFormula()
reconv = pycwmko.FromPyStore(workingContext, pyf)
reconv.run()
"""
elif arg == '-sparqlServer':
from swap.sparql import webserver
from swap import cwm_sparql
sandBoxed(True)
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
def _handler(s):
return cwm_sparql.sparql_queryString(workingContext, s)
webserver.sparql_handler = _handler
webserver.run()
elif arg == "-lxkbdump": # just for debugging
raise NotImplementedError
elif arg == "-lxfdump": # just for debugging
raise NotImplementedError
elif _lhs == "-prove":
# code copied from -filter without really being understood -sdh
_tmpstore = llyn.RDFStore(_outURI + "#_g",
metaURI=_metaURI,
argv=option_with,
crypto=option_crypto)
tmpContext = _tmpstore.newFormula(_uri + "#_formula")
_newURI = join(_baseURI, "_w_" + ` _genid `) # Intermediate
_genid = _genid + 1
_newContext = _tmpstore.newFormula(_newURI + "#_formula")
_tmpstore.loadURI(_uri)
print targetkb
elif arg == "-flatten":
#raise NotImplementedError
from swap import reify
workingContext = reify.flatten(workingContext)
elif arg == "-unflatten":
from swap import reify
workingContext = reify.unflatten(workingContext)
#raise NotImplementedError
elif arg == "-reify":
from swap import reify
workingContext = reify.reify(workingContext)
elif arg == "-dereify":
from swap import reify
workingContext = reify.dereify(workingContext)
elif arg == "-size":
progress("Size: %i statements in store, %i in working formula." %
(_store.size, workingContext.size()))
elif arg == "-strings": # suppress output
workingContext.outputStrings()
option_outputStyle = "-no"
elif arg == '-sparqlResults':
from cwm_sparql import outputString, SPARQL_NS
ns = _store.newSymbol(SPARQL_NS)
if not sparql_query_formula:
raise ValueError('No query')
else:
sys.stdout.write(
outputString(sparql_query_formula,
workingContext)[0].encode('utf_8'))
option_outputStyle = "-no"
elif arg == "-no": # suppress output
option_outputStyle = arg
elif arg[:8] == "-outURI=":
pass
elif arg == "-with":
break
else:
progress("cwm: Unknown option: " + arg)
sys.exit(-1)
# Squirt it out if not piped
workingContext.stayOpen = 0 # End its use as an always-open knoweldge base
if option_pipe:
workingContext.endDoc()
else:
if hasattr(_outSink, "serializeKB"):
raise NotImplementedError
else:
if verbosity() > 5: progress("Begining output.")
workingContext = workingContext.close()
assert workingContext.canonical != None
if option_outputStyle == "-ugly":
_store.dumpChronological(workingContext, _outSink)
elif option_outputStyle == "-bySubject":
_store.dumpBySubject(workingContext, _outSink)
elif option_outputStyle == "-no":
pass
elif option_outputStyle == "-debugString":
print workingContext.debugString()
else: # "-best"
_store.dumpNested(workingContext,
_outSink,
flags=option_flags[option_format])
def trueEasyMatches(self, s, n1, n2, newBindings=BindingTree(), boring=True):
## progress('easymatches on %s and %s' % (n1, n2))
if isinstance(n1, set):
n1 = frozenset(n1)
if isinstance(n2, set):
n2 = frozenset(n2)
if n1 in s.map:
progress('Already known case')
return s.map[n1] == n2
if not boring:
newBindings.int_and([(n1, n2)])
progress('newBindings=%s' % newBindings)
if isLiteral(n1) and isLiteral(n2):
return n1 == n2
if isSymbol(n1) and isSymbol(n2):
return n1 == n2
if isExistential(n1) and isExistential(n2):
return True
if isList(n1) and isList(n2):
n3 = n1.first
n4 = n2.first
if self(s, n3, n4, newBindings, False):
return self(s, n1.rest, n2.rest, newBindings)
if isFormula(n1) and isFormula(n2):
### This belongs in hard only
return True
if isSet(n1) and isSet(n2):
### set matching
### This is annoying. We mostly will try to do anything to
### avoid this
###
progress('Starting set matching on %s=?%s' % (n1, n2))
## first test length
if len(n1) != len(n2):
progress('Different sizes!')
return False
## easy cases are easy
if len(n1) == 1:
progress('Size 1 --- easy')
return self(s,
iter(n1).next(),
iter(n2).next(), newBindings, False)
## knock out everything we know
for x1 in n1:
if x1 in s.map and s.map[x1] not in n2:
progress('map failed --- s[%s] == %s, couldn\'t find it' %
(x1, s.map[x1]))
return False
n1 = set([x for x in n1 if x not in s.map])
n2 = set([x for x in n2 if x not in s.reverseMap])
## redundant, hopefully
if len(n1) != len(n2):
progress('How did I get here?')
return False
## sort into classes
tests = isLiteral, isSymbol, isExistential, isUniversal, isList, isFormula, isSet
g1 = tuple([set() for x in tests])
g2 = tuple([set() for x in tests])
for g, n3 in (g1, n1), (g2, n2):
for x in n3:
for i in range(len(tests)):
test = tests[i]
if test(x):
g[i].add(x)
break
else:
raise TypeError(x)
progress('Sort got us %s,%s' % (g1, g2))
## make sure every class works
if [len(x) for x in g1] != [len(x) for x in g2]:
progress('One class had a length mismatch!')
return False
## literals and symbols must match themselves
for s1, s2 in (g1[0], g2[0]), (g1[1], g2[1]):
for x in s1:
if x not in s2:
progress("Failed to find `%s' in s2=%s" % (x, s2))
return False
newBindings.int_and([(x, x)])
## rest are harder to do
## do we ever want to enumerate this?
for s1, s2 in zip(g1[2:], g2[2:]):
b = []
s1 = list(s1)
for k in orderings(list(s2)):
b.append(zip(s1, k))
newBindings.int_or(b)
progress('After that, newBindings=%s' % newBindings)
return True
return False
def nailFormula(f, assumptions=Set()):
"""Smush the formula.
Build a dictionary of nodes which are indirectly identified
by [inverse] functonal properties."""
global verbose
cc, predicates, ss, oo = getParts(f)
nodes = ss | oo
sofar = {}
bnodes = Set()
for node in nodes:
if node.generated() or node in f.existentials():
bnodes.add(node)
if verbose >=5: progress("Blank node: %s" % `node`)
else:
if verbose >=5: progress("Fixed node: %s" % `node`)
sofar[node] = []
meta = lookUp(predicates, assumptions)
ifps = predicates & Set(meta.each(pred=RDF.type, obj=OWL.InverseFunctionalProperty))
fps = predicates & Set(meta.each(pred=RDF.type, obj=OWL.FunctionalProperty))
if verbose > 1:
for p in fps: progress("Functional Property:", p)
for p in ifps: progress("Inverse Functional: ", p)
definitions = []
if len(bnodes) == 0:
if verbose > 0: progress("No bnodes in graph")
return bnodes, definitions
a = float(len(bnodes))/len(nodes)
if verbose > 1: progress("Proportion of bodes which are blank: %f" % a)
# if a == 0: return bnodes, definitions
loose = bnodes.copy()
equivs = Set()
# Note possible optmization: First pass only like this,
# future passes work from newNodes.
while loose:
newNailed = Set()
for preds, inverse, char in ((fps, 0, "!"), (ifps, 1, "^")):
for pred in preds:
if verbose > 3: progress("Predicate", pred)
ss = f.statementsMatching(pred=pred)
for s in ss:
if inverse: y, x = s.object(), s.subject()
else: x, y = s.object(), s.subject()
if not x.generated(): continue # Only anchor bnodes
if y not in loose: # y is the possible anchor
defi = (x, inverse, pred, y)
if x in loose: # this node
if verbose > 4: progress(" Nailed %s as %s%s%s" % (`x`, `y`, `char`, `pred`))
loose.discard(x)
newNailed.add(x)
else:
if verbose >=6 : progress(
" (ignored %s as %s%s%s)" % (`x`, `y`, `char`, `pred`))
definitions.append(defi)
# if verbose > 9: progress(" Definition[x] is now", definition[x])
if inverse: equivalentSet = Set(f.each(obj=y, pred=pred))
else: equivalentSet = Set(f.each(subj=y, pred=pred))
if len(equivalentSet) > 1: equivs.add(equivalentSet)
if not newNailed:
if verbose > 1:
progress("Failed to nail nodes:", loose)
if verbose > 3:
for n in loose:
debugBnode(n, f)
break
# At this point if we still have loose nodes, we have failed with ifps and fps.
# Diff may not be strong. (It might still be: the diffs might not involve weak definitions)
weak = loose.copy() # Remember
if verbose > 0: progress("\nFailed to directly nail everything, looking for weak nailings.")
# sys.exit(-1) #@@@
while loose:
newNailed = Set()
if verbose>2:
progress()
progress("Pass: loose = %s" % loose)
for x in loose.copy():
if verbose>3: progress("Checking weakly node %s" % x)
for s in f.statementsMatching(obj=x):
pred, y = s.predicate(), s.subject()
if y in loose:
if verbose > 4: progress("Can't nail to loose %s" % y)
continue # Can't nail to something loose
others = f.each(subj=y, pred=pred)
# @@ Should ignore redirected equivalent nodes in others
if len(others) != 1:
if verbose>4: progress("Can't nail: %s all are %s of %s." % (others, pred, y))
continue # Defn would be ambiguous in this graph
defi = (x, 0, pred, y)
if verbose >4: progress(" Weakly-nailed %s as %s%s%s" % (x, y, "!", pred))
loose.discard(x)
newNailed.add(x)
definitions.append(defi)
break # progress
else:
for s in f.statementsMatching(subj=x):
pred, obj = s.predicate(), s.object()
if obj in loose:
if verbose >4: progress("Can't nail to loose %s" % obj)
continue # Can't nail to something loose
others = f.each(obj=obj, pred=pred)
# @@ Should ignore redirected equivalent nodes in others
if len(others) != 1:
if verbose >4: progress(
"Can't nail: %s all have %s of %s." % (others, pred, obj))
continue # Defn would be ambiguous in this graph
defi = (x, 1, pred, obj)
if verbose>2: progress(" Weakly-nailed %s as %s%s%s" % (`x`, `obj`, "^", `pred`))
loose.discard(x)
newNailed.add(x)
definitions.append(defi)
break # progress
if not newNailed:
if verbose>0:
progress("Failed to even weakly nail nodes:", loose)
for n in loose:
progress("For node %s" % n)
for s in f.statementsMatching(subj=n):
progress(" %s %s; # could be ifp?" %(`s.predicate()`, `s.object()`))
for s in f.statementsMatching(obj=n):
progress(" is %s of %s; # could be fp?" %(s.predicate(), s.subject()))
raise ValueError("Graph insufficiently labelled for nodes: %s" % loose)
if verbose>0 and not weak: progress("Graph is solid.")
if verbose>0 and weak: progress("Graph is NOT solid.")
f.reopen()
for es in equivs:
if verbose>3: progress("Equivalent: ", es)
prev = None
for x in es:
if prev:
f.add(x, OWL.sameAs, prev)
prev = x
return bnodes, definitions
def main():
global already, agenda, errors
parseAs = None
grammarFile = None
parseFile = None
yaccFile = None
global verbose
global g
verbose = 0
lumped = 1
try:
opts, args = getopt.getopt(sys.argv[1:], "ha:v:p:g:y:",
["help", "as=", "verbose=", "parse=", "grammar=", "yacc="])
except getopt.GetoptError:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose =int(a)
diag.chatty_flag = int(a)
if o in ("-a", "--as"):
parseAs = uripath.join(uripath.base(), a)
if o in ("-p", "--parse"):
parseFile = uripath.join(uripath.base(), a)
if o in ("-g", "--grammar"):
grammarFile = uripath.join(uripath.base(), a)
if o in ("-y", "--yacc"):
yaccFile = uripath.join(uripath.base(), a)[5:] # strip off file:
# if testFiles == []: testFiles = [ "/dev/stdin" ]
if not parseAs:
usage()
sys.exit(2)
parseAs = uripath.join(uripath.base(), parseAs)
if not grammarFile:
grammarFile = parseAs.split("#")[0] # strip off fragid
else:
grammarFile = uripath.join(uripath.base(), grammarFile)
# The Grammar formula
progress("Loading " + grammarFile)
start = clock()
g = load(grammarFile)
taken = clock() - start + 1
progress("Loaded %i statements in %fs, ie %f/s." %
(len(g), taken, len(g)/taken))
document = g.newSymbol(parseAs)
already = []
agenda = []
errors = []
doProduction(document)
while agenda:
x = agenda[0]
agenda = agenda[1:]
already.append(x)
doProduction(x)
if errors != []:
progress("###### FAILED with %i errors." % len(errors))
for s in errors: progress ("\t%s" % s)
exit(-2)
else:
progress( "Ok for predictive parsing")
#if parser.verb: progress "Branch table:", branchTable
if verbose:
progress( "Literal terminals: %s" % literalTerminals.keys())
progress("Token regular expressions:")
for r in tokenRegexps:
progress( "\t%s matches %s" %(r, tokenRegexps[r].pattern) )
if yaccFile:
yacc=open(yaccFile, "w")
yaccConvert(yacc, document, tokenRegexps)
yacc.close()
if parseFile == None: exit(0)
ip = webAccess.urlopenForRDF(parseFile, None)
str = ip.read().decode('utf_8')
sink = g.newFormula()
keywords = g.each(pred=BNF.keywords, subj=document)
keywords = [a.value() for a in keywords]
p = PredictiveParser(sink=sink, top=document, branchTable= branchTable,
tokenRegexps= tokenRegexps, keywords = keywords)
p.verb = verbose
start = clock()
p.parse(str)
taken = clock() - start + 1
progress("Loaded %i chars in %fs, ie %f/s." %
(len(str), taken, len(str)/taken))
progress("Parsed <%s> OK" % parseFile)
sys.exit(0) # didn't crash
def __init__(self, s, level=0):
self._s = s
if True or chatty > 0:
progress(" "*(level*4), "Proof failed: ", s)
def main():
global verbose, proofs, chatty, normal, no_action
start = 1
cwm_command = '../cwm.py'
python_command = 'python -tt'
global ploughOn # even if error
ploughOn = 0
global verbose
verbose = 0
global just_fix_it
just_fix_it = 0
if diag.print_all_file_names:
a = file('testfilelist', 'w')
a.write('')
a.close()
try:
opts, testFiles = getopt.getopt(sys.argv[1:], "h?s:nNcipf:v", [
"help", "start=", "testsFrom=", "no-action", "No-normal", "chatty",
"ignoreErrors", "proofs", "verbose", "overwrite", "cwm="
])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "-?", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose = 1
if o in ("-i", "--ignoreErrors"):
ploughOn = 1
if o in ("-s", "--start"):
start = int(a)
if o in ("-f", "--testsFrom"):
testFiles.append(a)
if o in ("-n", "--no-action"):
no_action = 1
if o in ("-N", "--No-normal"):
normal = 0
if o in ("-c", "--chatty"):
chatty = 1
if o in ("-p", "--proofs"):
proofs = 1
if o in ("--overwrite", ):
just_fix_it = 1
if o in ("--cwm", "--the_end"):
cwm_command = a
assert system("mkdir -p ,temp") == 0
assert system("mkdir -p ,diffs") == 0
if proofs: assert system("mkdir -p ,proofs") == 0
tests = 0
passes = 0
global problems
problems = []
REFWD = "http://example.com/swap/test"
WD = base()[:-1]
#def basicTest(case, desc, args)
if verbose: progress("Test files:", testFiles)
kb = loadMany(testFiles, referer="")
testData = []
RDFTestData = []
RDFNegativeTestData = []
perfData = []
n3PositiveTestData = []
n3NegativeTestData = []
sparqlTestData = []
# for fn in testFiles:
# print "Loading tests from", fn
# kb=load(fn)
for t in kb.each(pred=rdf.type, obj=test.CwmTest):
verboseDebug = kb.contains(subj=t, pred=rdf.type, obj=test.VerboseTest)
u = t.uriref()
ref = kb.the(t, test.referenceOutput)
if ref == None:
case = str(kb.the(t, test.shortFileName))
refFile = "ref/%s" % case
else:
refFile = refTo(base(), ref.uriref())
case = ""
for ch in refFile:
if ch in "/#": case += "_"
else: case += ch # Make up test-unique temp filename
def main(argv):
global chatty
global debugLevelForInference
global debugLevelForParsing
global nameBlankNodes
setVerbosity(0)
policy = ParsingOK()
try:
opts, args = getopt.getopt(argv[1:], "hv:c:p:B:a", [
"help", "verbose=", "chatty=", "parsing=", "nameBlankNodes",
"allPremises", "profile", "report"
])
except getopt.GetoptError:
sys.stderr.write("check.py: Command line syntax error.\n\n")
usage()
sys.exit(2)
output = None
report = False
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
chatty = int(a)
if o in ("-p", "--verboseParsing"):
debugLevelForParsing = int(a)
if o in ("-c", "--chatty"):
debugLevelForInference = int(a)
if o in ("-B", "--nameBlankNodes"):
nameBlankNodes = 1
if o in ("-a", "--allPremises"):
policy = AllPremises()
if o in ("--profile"):
pass
if o in ("--report"):
report = True
if nameBlankNodes: flags = "B"
else: flags = ""
if args:
fyi("Reading proof from " + args[0])
proof = topLevelLoad(args[0], flags=flags)
else:
fyi("Reading proof from standard input.", thresh=5)
proof = topLevelLoad(flags=flags)
# setVerbosity(60)
fyi("Length of proof formula: " + ` len(proof) `, thresh=5)
try:
c = Checker(proof)
if report:
sys.stdout.write(PfReportHeader)
c.report(sys.stdout)
sys.stdout.write("\n\nConclusion::\n\n")
proved = c.result(c.conjecture()[1], policy=policy)
fyi("Proof looks OK. %i Steps" % proofSteps, thresh=5)
setVerbosity(0)
txt = proved.n3String().encode('utf-8')
print "\n".join([' ' + ln.strip() for ln in txt.split("\n")])
except InvalidProof, e:
progress("Proof invalid:", e)
sys.exit(-1)
def trueEasyMatches(self, s, n1, n2, newBindings=BindingTree(), boring=True):
## progress('easymatches on %s and %s' % (n1, n2))
if isinstance(n1, set):
n1 = frozenset(n1)
if isinstance(n2, set):
n2 = frozenset(n2)
if n1 in s.map:
progress("Already known case")
return s.map[n1] == n2
if not boring:
newBindings.int_and([(n1, n2)])
progress("newBindings=%s" % newBindings)
if isLiteral(n1) and isLiteral(n2):
return n1 == n2
if isSymbol(n1) and isSymbol(n2):
return n1 == n2
if isExistential(n1) and isExistential(n2):
return True
if isList(n1) and isList(n2):
n3 = n1.first
n4 = n2.first
if self(s, n3, n4, newBindings, False):
return self(s, n1.rest, n2.rest, newBindings)
if isFormula(n1) and isFormula(n2):
### This belongs in hard only
return True
if isSet(n1) and isSet(n2):
### set matching
### This is annoying. We mostly will try to do anything to
### avoid this
###
progress("Starting set matching on %s=?%s" % (n1, n2))
## first test length
if len(n1) != len(n2):
progress("Different sizes!")
return False
## easy cases are easy
if len(n1) == 1:
progress("Size 1 --- easy")
return self(s, iter(n1).next(), iter(n2).next(), newBindings, False)
## knock out everything we know
for x1 in n1:
if x1 in s.map and s.map[x1] not in n2:
progress("map failed --- s[%s] == %s, couldn't find it" % (x1, s.map[x1]))
return False
n1 = set([x for x in n1 if x not in s.map])
n2 = set([x for x in n2 if x not in s.reverseMap])
## redundant, hopefully
if len(n1) != len(n2):
progress("How did I get here?")
return False
## sort into classes
tests = isLiteral, isSymbol, isExistential, isUniversal, isList, isFormula, isSet
g1 = tuple([set() for x in tests])
g2 = tuple([set() for x in tests])
for g, n3 in (g1, n1), (g2, n2):
for x in n3:
for i in range(len(tests)):
test = tests[i]
if test(x):
g[i].add(x)
break
else:
raise TypeError(x)
progress("Sort got us %s,%s" % (g1, g2))
## make sure every class works
if [len(x) for x in g1] != [len(x) for x in g2]:
progress("One class had a length mismatch!")
return False
## literals and symbols must match themselves
for s1, s2 in (g1[0], g2[0]), (g1[1], g2[1]):
for x in s1:
if x not in s2:
progress("Failed to find `%s' in s2=%s" % (x, s2))
return False
newBindings.int_and([(x, x)])
## rest are harder to do
## do we ever want to enumerate this?
for s1, s2 in zip(g1[2:], g2[2:]):
b = []
s1 = list(s1)
for k in orderings(list(s2)):
b.append(zip(s1, k))
newBindings.int_or(b)
progress("After that, newBindings=%s" % newBindings)
return True
return False
def differences(f, g, assumptions):
"""Smush the formulae. Compare them, generating patch instructions."""
global lumped
# Cross-map nodes:
g_bnodes, g_definitions = nailFormula(g, assumptions)
bnodes, definitions = nailFormula(f, assumptions)
if verbose > 1: progress("\n Done nailing")
definitions.reverse() # go back down list @@@ reverse the g list too? @@@
g_definitions.reverse() # @@ needed for the patch generation
unmatched = bnodes.copy()
match = {} # Mapping of nodes in f to nodes in g
for x, inverse, pred, y in definitions:
if x in match: continue # done already
if x in f._redirections:
if verbose > 3: progress("Redirected %s to %s. Ignoring" % (`x`, `f._redirections[x]`))
unmatched.discard(x)
continue
if verbose > 3: progress("Definition of %s = %s%s%s"% (`x`, `y` , ".!^"[inverse], `pred`))
if y.generated():
while y in f._redirections:
y = f._redirections[y]
if verbose>4: progress(" redirected to %s = %s%s%s"% (`x`, `y`, "!^"[inverse], `pred`))
yg = match.get(y, None)
if yg == None:
if verbose>4: progress(" Had definition for %s in terms of %s which is not matched"%(`x`,`y`))
continue
else:
yg = y
if inverse: # Inverse functional property like ssn
matches = Set(g.each(obj=yg, pred=pred))
else: matches = Set(g.each(subj=yg, pred=pred))
if len(matches) == 0:
continue # This is normal - the node does not exist in the other graph
# raise RuntimeError("Can't match %s" % x)
if len(matches) > 1:
raise RuntimeError("""Rats. Wheras in the first graph %s%s%s uniquely selects %s,
in the other graph there are more than 1 matches: %s""" % (`y`, "!^"[inverse], `pred`, `x`, `matches`))
for q in matches: # pick only one @@ python function?
z = q
break
if verbose > 2:
progress("Found match for %s in %s " % (`x`,`z`))
match[x] = z
unmatched.discard(x)
if len(unmatched) > 0:
if verbose >1:
progress("Failed to match all nodes:", unmatched)
for n in unmatched:
debugBnode(n, f)
# Find common parts
only_f, only_g = removeCommon(f,g, match)
delta = f.newFormula()
if len(only_f) == 0 and len(only_g) == 0:
return delta
f = f.close() # We are not going to mess with them any more
g = g.close()
common = Set([match[x] for x in match])
if verbose>2: progress("Common bnodes (as named in g)", common)
patches(delta, f, only_f, Set(), definitions, deleting=1)
patches(delta, g, only_g, common, g_definitions, deleting=0)
if lumped:
consolidate(delta, delta.store.insertion)
consolidate(delta, delta.store.deletion)
return delta
def main():
oldFiles = []
changedFiles = []
commonFiles = []
assumptions = Set()
global ploughOn # even if error
ploughOn = 0
global verbose
global lumped
verbose = 0
lumped = 1
try:
opts, args = getopt.getopt(sys.argv[1:], "hf:t:c:m:vg",
["help", "from=", "to=", "common=", "meta=", "verbose", "granularity="])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose += 1
if o in ("-l", "--granularity"):
lumped = int(a)
if o in ("-f", "--from"):
oldFiles.append(a)
if o in ("-t", "--to"):
changedFiles.append(a)
if o in ("-m", "--meta"):
assumptions.add(a)
if o in ("-o", "--out"):
output = a
version = "$Id$"[1:-1]
if len(oldFiles) != 1 or len(changedFiles) < 1:
usage()
sys.exit(2)
G = loadFiles(oldFiles)
commonFile = oldFiles[0]
deltas = []
changedFormulas = []
biggest = None
for i in range(len(changedFiles)):
changedFile = changedFiles[i]
progress("Diffing %s against %s" %(changedFile, commonFile))
F = loadFiles([changedFile])
D = differences(F, G, assumptions):
deltas.append(D)
if len(D) == 0:
progress("Great, no change in %s." % (changedFile))
else:
if biggest == None or len(D) > len(biggest)):
biggest_i = i
biggest = F
if biggest == None:
progress("No changes in any inputs.")
if outputFile != None:
sys.exit(0)
# exit without overwriting
else:
progress("Most changed is %s" %(changedFiles[biggest_i]))
G = biggest # (drop the original common formula)
for i in range(len(changedFiles)):
progress("Applying changes from %s" %(changedFiles[i]))
G = patch(G, delta[i])
if outputFile:
out = open(outputFile, "w")
out.write(G.asN3String())
close(out)
else:
print G.asN3String()
sys.exit(0) # didn't crash
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
"""Fin - financial summary
<command> <options> <inputURIs>
Totals transactions by classes to which they are known to belong
This is or was http://www.w3.org/2000/10/swap/pim/fin.py
"""
#import urllib
#import time
import sys
# global sax2rdf
global kb, tax
def noteError(e):
if not errors.get(s, None): errors[s] = [];
errors[s].append(e)
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
fatalErrors = 0;
# Load the data:
kb = loadMany(inputURIs)
qu_date = qu.date
qu_in_USD = qu.in_USD
qu_amount = qu.amount
qu_payee = qu.payee
qu_Classified = qu.Classified
qu_Unclassified = qu.Unclassified
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
if verbose:
progress("Tax categories" + `taxCategories`)
specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]
####### Analyse the data:
numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
monthTotals = [0] * numberOfMonths
incomeByMonth = [0] * numberOfMonths
income, outgoings = 0,0
outgoingsByMonth = [0] * numberOfMonths
quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
bottomCategories = [];
for c in quCategories:
if isBottomClass(c): bottomCategories.append(c);
totals = {} # Total by all classes of transaction
count = {} # Number of transactions
byMonth = {}
sts = kb.statementsMatching(pred=qu.amount) # Ideally one per transaction
errors = {}
for st in sts:
s = st.subject()
uri = s.uriref()
# classified = kb.each(pred=rdf_type, obj=qu_Classified)
# unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
# for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
# for s in classified + unclassified:
# progress( "Transaction ", `s`)
t_ok, c_ok = 0, 0
cats = allClasses(kb.each(subj=s, pred=rdf.type))
# progress( "Categories: "+`cats`)
month = monthNumber(s)
if month not in range(numberOfMonths) : continue
payees = kb.each(subj=s, pred=qu_payee)
if not payees:
progress("@@@ Error: No payee for "+`uri`)
payee = "@@@@@@ Unknown";
fatalErrors += 1;
elif len(payees) >1 and str(payees[0]) == "Check":
payee = payees[1]
else:
payee = payees[0]
amounts = kb.each(subj=s, pred=qu_in_USD)
if len(amounts) == 0:
amounts = kb.each(subj=s, pred=qu_amount)
if len(amounts) == 0:
progress("@@@ Error: No USD amount for "+`uri`)
fatalErrors += 1;
else:
progress("Warning: No USD amount for "+`uri`+", assuming USD")
if len(amounts) >1:
if (cat_ns.Internal not in cats or
len(amounts) != 2 ):
fatalErrors += 1;
progress(
"Error: More than one USD amount %s for transaction %s -- ignoring!\n"
% (`amounts`,uri))
else:
sum = float(amounts[0]) + float(amounts[1])
if sum != 0:
fatalErrors += 1;
progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
% (sum, amounts, uri))
continue
if len(amounts) != 1:
progress("@@@ Error: No amount for "+`uri`);
fatalErrors += 1;
ss = kb.statementsMatching(subj=s)
progress(`ss`+'; KB='+`kb.n3String()`)
continue
amount = float(amounts[0].__str__())
# print "%s %40s %10s month %i" %(date, payee, `amount`, month)
monthTotals[month] = monthTotals[month] + amount
if cat_ns.Internal not in cats:
if amount > 0:
incomeByMonth[month] = incomeByMonth[month] + amount
income = income + amount
else:
outgoingsByMonth[month] = outgoingsByMonth[month] + amount
outgoings = outgoings + amount
normalCats = [] # For this item
for c in cats:
totals[c] = totals.get(c, 0) + amount
byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
count[c] = count.get(c, 0) + 1
byMonth[c][month] = byMonth[c][month] + amount
if c not in specialCategories:
normalCats.append(c)
bottomCats = normalCats[:] # Copy
for b in normalCats:
sups = kb.each(subj=b, pred=rdfs.subClassOf)
for sup in sups:
if sup in bottomCats:
bottomCats.remove(sup)
if len(bottomCats) == 0:
noteError("No categoriy: %s for <%s>" # all cats: %s, raw cats:%s"
%(`cats`, `s`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
noteError("Be more specifc: %s for <%s>" %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
if len(bottomCats) > 1:
noteError("Inconsistent categories: %s" # all cats: %s, raw cats:%s"
%(`bottomCats`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
print '<html xmlns="http://www.w3.org/1999/xhtml">'
if '--summary' in sys.argv:
title = "Monthly summary"
elif '--issues' in sys.argv:
title = "Issues"
else:
title = "Report"
print """<head>
<meta charset='UTF-8'>
<title>%s</title>
<link rel="Stylesheet" href="report.css">
</head>
<body>
""" % (title)
# <img src="sand-dollar.gif" alt="dollar" align="right"/>
version = "$Id: fin.py,v 1.31 2014-07-23 13:23:05 timbl Exp $"
# SUMMARY TABLE OF CATEGORY BY MONTH
if '--summary' in sys.argv:
print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
print "<tr><th></th><th>Total </th>"
for month in range(numberOfMonths):
m = month + int(startDate[5:7]) - 1
while m > 11: m -= 12 # Modulo in python?
print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
print "</tr>"
def listFor(c, depth=0): # Any, because there could be 2 copies of same list :-(
subs = kb.any(subj = c, pred = owl.disjointUnionOf);
res = [ (c, depth) ];
if subs == None:
subs = kb.each(pred = rdfs.subClassOf, obj = c);
if len(subs) > 0:
sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
for sub in subs: res += listFor(sub, depth+1)
else:
for sub in subs: res += listFor(sub, depth+1)
return res
printOrder = listFor(qu.Transaction);
for cat, depth in printOrder:
label = kb.the(subj=cat, pred=rdfs.label)
if label == None:
label = `cat`
sys.stderr.write("@@ No label for "+`cat` +"\n")
else:
label = str(label)
anchor = cat.fragid
if totals.get(cat, None) != None:
print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
numberOfMonths, indent = depth)
print "<tr><td colspan='14'> ___ </td></tr>"
print monthGridRow("Income", None, income, incomeByMonth, numberOfMonths)
print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)
print "</table>"
# Chart of income stacked up against expenses
if '--charts' in sys.argv:
print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"
writeChart(filename = "chart.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 1)
writeChart(filename = "all.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 0)
# Output totals
if (totalsFilename):
ko = kb.newFormula()
for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
ko.close()
fo = open(totalsFilename, "w")
fo.write(ko.n3String())
fo.close
if '--issues' in sys.argv:
# Generate a list of errors found
errstr = ""
for x, list in errors.items():
errstr += transactionRow(x)
for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n" # @@@ encode error string
if errstr:
print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
# List Unclassified Income and Spending
def transactionList(cat):
ts = kb.each(pred = rdf.type, obj = cat)
if len(ts) == 0: return ""
label = kb.any(cat, rdfs.label)
st = '<h2>'+label.value()+'</h2>\n'
return st + transactionTable(ts)
for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
print transactionList(cat)
print reimbursablesCheck();
internalCheck()
if 0:
print "<h2>Tax Categories</h2>"
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)
print "<h2>Tax stuff</h2>"
print "<table>"
print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
print "</table>"
# print "<h2>Personal Category total</h2>"
# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)
print
print "Note totals for tax and personal breakdowns must match."
dates = kb.statementsMatching(pred=qu.date)
print "There should be a total of %i transactions in each." % len(dates)
if 0:
print "<pre>(consistency check)"
problems = 0
for s in dates:
tra = s.subject()
types = kb.each(subj=tra, pred=rdf_type)
for typ in types:
if typ is qu.Unclassified or typ is qu.Classified:
break # ok
else:
print "@@@@ problem transcation with no classified or unclassified, with types", types
printTransactionDetails(tra)
problems = problems + 1
print problems, "problems.</pre>"
print "</body></html>"
return fatalErrors