def loadFiles(files):
graph = myStore.formula()
graph.setClosureMode("e") # Implement sameAs by smushing
if verbose>0: progress("Loading %s..." % files)
graph = myStore.loadMany(files, openFormula=graph)
if verbose>0: progress("Loaded", graph)
return graph
def main():
"""The main function
""" ### """
try:
opts, testFiles = getopt.getopt(sys.argv[1:], "hc:o:e:",
["help", "command=", "output=", "error="])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
errorFile = ',temp/__error.txt' # was "/dev/null"
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-c", "--command"):
commandFile = a
if o in ("-o", "--output"):
outputFile = a
if o in ("-e", "--error"):
errorFile = a
assert system("mkdir -p ,temp") == 0
assert system("mkdir -p ,diffs") == 0
testFiles = testFiles + [
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql1/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql2/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql3/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql4/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bnode-coreference/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/boolean-effective-value/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/construct/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/regex/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/sort/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/triple-match/manifest.ttl',
'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/type-promotion/manifest.ttl']
kb = loadMany(testFiles, referer="")
output = formula()
testCwmSparql(kb, output, errorFile)
output.close()
output.store.dumpNested(output, ToN3(file(outputFile, 'w').write))
def lookUp(predicates, assumptions=Set()):
"""Look up all the schemas for the predicates given"""
global verbose
schemas = assumptions
for pred in predicates:
if verbose > 3: progress("Predicate: %s" % `pred`)
u = pred.uriref()
hash = u.find("#")
if hash <0:
if verbose > 1: progress("Warning: Predicate <%s> looks like web resource not Property" % u)
else:
schemas.add(u[:hash])
if verbose > 2:
for r in schemas:
progress("Metadata to be loaded: ", r)
if schemas:
return loadMany([(x) for x in schemas])
return myStore.store.newFormula() # Empty formula
def main():
"""The main function
""" ### """
try:
opts, testFiles = getopt.getopt(sys.argv[1:], "hc:o:e:",
["help", "command=", "output=", "error="])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
errorFile = ',temp/__error.txt' # was "/dev/null"
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-c", "--command"):
commandFile = a
if o in ("-o", "--output"):
outputFile = a
if o in ("-e", "--error"):
errorFile = a
commands = [w[:-1] for w in file(commandFile, 'r')]
assert system("mkdir -p ,temp") == 0
assert system("mkdir -p ,diffs") == 0
kb = loadMany(testFiles, referer="")
z = 0
for command in commands:
output = formula()
testParser(command, kb, output, errorFile)
output.close()
output.store.dumpNested(output, ToN3(file(outputFile+`z`, 'w').write))
z = z+1
def loadFiles(files):
graph = myStore.formula()
graph.setClosureMode("e") # Implement sameAs by smushing
graph = myStore.loadMany(files, openFormula=graph)
if verbose: progress("Loaded", graph, graph.__class__)
return graph
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
"""Fin - financial summary
<command> <options> <inputURIs>
Totals transactions by classes to which they are known to belong
This is or was http://www.w3.org/2000/10/swap/pim/fin.py
"""
#import urllib
#import time
import sys
# global sax2rdf
global kb, tax
def noteError(e):
if not errors.get(s, None): errors[s] = [];
errors[s].append(e)
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
fatalErrors = 0;
# Load the data:
kb = loadMany(inputURIs)
qu_date = qu.date
qu_in_USD = qu.in_USD
qu_amount = qu.amount
qu_payee = qu.payee
qu_Classified = qu.Classified
qu_Unclassified = qu.Unclassified
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
if verbose:
progress("Tax categories" + `taxCategories`)
specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]
####### Analyse the data:
numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
monthTotals = [0] * numberOfMonths
incomeByMonth = [0] * numberOfMonths
income, outgoings = 0,0
outgoingsByMonth = [0] * numberOfMonths
quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
bottomCategories = [];
for c in quCategories:
if isBottomClass(c): bottomCategories.append(c);
totals = {} # Total by all classes of transaction
count = {} # Number of transactions
byMonth = {}
sts = kb.statementsMatching(pred=qu.amount) # Ideally one per transaction
errors = {}
for st in sts:
s = st.subject()
uri = s.uriref()
# classified = kb.each(pred=rdf_type, obj=qu_Classified)
# unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
# for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
# for s in classified + unclassified:
# progress( "Transaction ", `s`)
t_ok, c_ok = 0, 0
cats = allClasses(kb.each(subj=s, pred=rdf.type))
# progress( "Categories: "+`cats`)
month = monthNumber(s)
if month not in range(numberOfMonths) : continue
payees = kb.each(subj=s, pred=qu_payee)
if not payees:
progress("@@@ Error: No payee for "+`uri`)
payee = "@@@@@@ Unknown";
fatalErrors += 1;
elif len(payees) >1 and str(payees[0]) == "Check":
payee = payees[1]
else:
payee = payees[0]
amounts = kb.each(subj=s, pred=qu_in_USD)
if len(amounts) == 0:
amounts = kb.each(subj=s, pred=qu_amount)
if len(amounts) == 0:
progress("@@@ Error: No USD amount for "+`uri`)
fatalErrors += 1;
else:
progress("Warning: No USD amount for "+`uri`+", assuming USD")
if len(amounts) >1:
if (cat_ns.Internal not in cats or
len(amounts) != 2 ):
fatalErrors += 1;
progress(
"Error: More than one USD amount %s for transaction %s -- ignoring!\n"
% (`amounts`,uri))
else:
sum = float(amounts[0]) + float(amounts[1])
if sum != 0:
fatalErrors += 1;
progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
% (sum, amounts, uri))
continue
if len(amounts) != 1:
progress("@@@ Error: No amount for "+`uri`);
fatalErrors += 1;
ss = kb.statementsMatching(subj=s)
progress(`ss`+'; KB='+`kb.n3String()`)
continue
amount = float(amounts[0].__str__())
# print "%s %40s %10s month %i" %(date, payee, `amount`, month)
monthTotals[month] = monthTotals[month] + amount
if cat_ns.Internal not in cats:
if amount > 0:
incomeByMonth[month] = incomeByMonth[month] + amount
income = income + amount
else:
outgoingsByMonth[month] = outgoingsByMonth[month] + amount
outgoings = outgoings + amount
normalCats = [] # For this item
for c in cats:
totals[c] = totals.get(c, 0) + amount
byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
count[c] = count.get(c, 0) + 1
byMonth[c][month] = byMonth[c][month] + amount
if c not in specialCategories:
normalCats.append(c)
bottomCats = normalCats[:] # Copy
for b in normalCats:
sups = kb.each(subj=b, pred=rdfs.subClassOf)
for sup in sups:
if sup in bottomCats:
bottomCats.remove(sup)
if len(bottomCats) == 0:
noteError("No categoriy: %s for <%s>" # all cats: %s, raw cats:%s"
%(`cats`, `s`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
noteError("Be more specifc: %s for <%s>" %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
if len(bottomCats) > 1:
noteError("Inconsistent categories: %s" # all cats: %s, raw cats:%s"
%(`bottomCats`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
print '<html xmlns="http://www.w3.org/1999/xhtml">'
if '--summary' in sys.argv:
title = "Monthly summary"
elif '--issues' in sys.argv:
title = "Issues"
else:
title = "Report"
print """<head>
<meta charset='UTF-8'>
<title>%s</title>
<link rel="Stylesheet" href="report.css">
</head>
<body>
""" % (title)
# <img src="sand-dollar.gif" alt="dollar" align="right"/>
version = "$Id$"
# SUMMARY TABLE OF CATEGORY BY MONTH
if '--summary' in sys.argv:
print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
print "<tr><th></th><th>Total </th>"
for month in range(numberOfMonths):
m = month + int(startDate[5:7]) - 1
while m > 11: m -= 12 # Modulo in python?
print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
print "</tr>"
def listFor(c, depth=0): # Any, because there could be 2 copies of same list :-(
subs = kb.any(subj = c, pred = owl.disjointUnionOf);
res = [ (c, depth) ];
if subs == None:
subs = kb.each(pred = rdfs.subClassOf, obj = c);
if len(subs) > 0:
sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
for sub in subs: res += listFor(sub, depth+1)
else:
for sub in subs: res += listFor(sub, depth+1)
return res
printOrder = listFor(qu.Transaction);
for cat, depth in printOrder:
label = kb.the(subj=cat, pred=rdfs.label)
if label == None:
label = `cat`
sys.stderr.write("@@ No label for "+`cat` +"\n")
else:
label = str(label)
anchor = cat.fragid
if totals.get(cat, None) != None:
print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
numberOfMonths, indent = depth)
print "<tr><td colspan='14'> ___ </td></tr>"
print monthGridRow("Income", None, income, incomeByMonth, numberOfMonths)
print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)
print "</table>"
# Chart of income stacked up against expenses
if '--charts' in sys.argv:
print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"
writeChart(filename = "chart.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 1)
writeChart(filename = "all.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 0)
# Output totals
if (totalsFilename):
ko = kb.newFormula()
for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
ko.close()
fo = open(totalsFilename, "w")
fo.write(ko.n3String())
fo.close
if '--issues' in sys.argv:
# Generate a list of errors found
errstr = ""
for x, list in errors.items():
errstr += transactionRow(x)
for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n" # @@@ encode error string
if errstr:
print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
# List Unclassified Income and Spending
def transactionList(cat):
ts = kb.each(pred = rdf.type, obj = cat)
if len(ts) == 0: return ""
label = kb.any(cat, rdfs.label)
st = '<h2>'+label.value()+'</h2>\n'
return st + transactionTable(ts)
for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
print transactionList(cat)
print reimbursablesCheck();
internalCheck()
if 0:
print "<h2>Tax Categories</h2>"
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)
print "<h2>Tax stuff</h2>"
print "<table>"
print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
print "</table>"
# print "<h2>Personal Category total</h2>"
# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)
print
print "Note totals for tax and personal breakdowns must match."
dates = kb.statementsMatching(pred=qu.date)
print "There should be a total of %i transactions in each." % len(dates)
if 0:
print "<pre>(consistency check)"
problems = 0
for s in dates:
tra = s.subject()
types = kb.each(subj=tra, pred=rdf_type)
for typ in types:
if typ is qu.Unclassified or typ is qu.Classified:
break # ok
else:
print "@@@@ problem transcation with no classified or unclassified, with types", types
printTransactionDetails(tra)
problems = problems + 1
print problems, "problems.</pre>"
print "</body></html>"
return fatalErrors
def main():
global verbose, proofs, chatty, normal, no_action
start = 1
cwm_command='../cwm.py'
python_command='python -tt'
global ploughOn # even if error
ploughOn = 0
global verbose
verbose = 0
global just_fix_it
just_fix_it = 0
if diag.print_all_file_names:
a = file('testfilelist','w')
a.write('')
a.close()
try:
opts, testFiles = getopt.getopt(sys.argv[1:], "h?s:nNcipf:v",
["help", "start=", "testsFrom=", "no-action", "No-normal", "chatty",
"ignoreErrors", "proofs", "verbose","overwrite","cwm="])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "-?", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose = 1
if o in ("-i", "--ignoreErrors"):
ploughOn = 1
if o in ("-s", "--start"):
start = int(a)
if o in ("-f", "--testsFrom"):
testFiles.append(a)
if o in ("-n", "--no-action"):
no_action = 1
if o in ("-N", "--No-normal"):
normal = 0
if o in ("-c", "--chatty"):
chatty = 1
if o in ("-p", "--proofs"):
proofs = 1
if o in ("--overwrite",):
just_fix_it = 1
if o in ("--cwm", "--the_end"):
cwm_command=a
assert system("mkdir -p ,temp") == 0
assert system("mkdir -p ,diffs") == 0
if proofs: assert system("mkdir -p ,proofs") == 0
tests=0
passes=0
global problems
problems = []
REFWD="http://example.com/swap/test"
WD = base()[:-1]
#def basicTest(case, desc, args)
if verbose: progress("Test files:", testFiles)
kb = loadMany(testFiles, referer="")
testData = []
RDFTestData = []
RDFNegativeTestData = []
perfData = []
n3PositiveTestData = []
n3NegativeTestData = []
sparqlTestData = []
# for fn in testFiles:
# print "Loading tests from", fn
# kb=load(fn)
for t in kb.each(pred=rdf.type, obj=test.CwmTest):
verboseDebug = kb.contains(subj=t, pred=rdf.type, obj=test.VerboseTest)
u = t.uriref()
ref = kb.the(t, test.referenceOutput)
if ref == None:
case = str(kb.the(t, test.shortFileName))
refFile = "ref/%s" % case
else:
refFile = refTo(base(), ref.uriref())
case = ""
for ch in refFile:
if ch in "/#": case += "_"
else: case += ch # Make up test-unique temp filename
description = str(kb.the(t, test.description))
arguments = str(kb.the(t, test.arguments))
environment = kb.the(t, test.environment)
if environment == None: env=""
else: env = str(environment) + " "
testData.append((t, t.uriref(), case, refFile, description, env, arguments, verboseDebug))
for t in kb.each(pred=rdf.type, obj=rdft.PositiveParserTest):
x = t.uriref()
y = x.find("/rdf-tests/")
x = x[y+11:] # rest
for i in range(len(x)):
if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
case = "rdft_" + x + ".nt" # Hack - temp file name
description = str(kb.the(t, rdft.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, rdft.inputDocument).uriref()
outputDocument = kb.the(t, rdft.outputDocument).uriref()
status = kb.the(t, rdft.status).string
good = 1
if status != "APPROVED":
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.ReificationTest:
if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
good = 0
## if cat is triage.ParseTypeLiteralTest:
## if verbose: print "\tNot supported (Parse type literal): "+ inputDocument[-40:]
## good = 0
if good:
RDFTestData.append((t.uriref(), case, description, inputDocument, outputDocument))
for t in kb.each(pred=rdf.type, obj=rdft.NegativeParserTest):
x = t.uriref()
y = x.find("/rdf-tests/")
x = x[y+11:] # rest
for i in range(len(x)):
if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
case = "rdft_" + x + ".nt" # Hack - temp file name
description = str(kb.the(t, rdft.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, rdft.inputDocument).uriref()
status = kb.the(t, rdft.status).string
good = 1
if status != "APPROVED":
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.knownError:
if verbose: print "\tknown failure: "+ inputDocument[-40:]
good = 0
if cat is triage.ReificationTest:
if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
good = 0
if good:
RDFNegativeTestData.append((t.uriref(), case, description, inputDocument))
for t in kb.each(pred=rdf.type, obj=n3test.PositiveParserTest):
u = t.uriref()
hash = u.rfind("#")
slash = u.rfind("/")
assert hash >0 and slash > 0
case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
description = str(kb.the(t, n3test.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, n3test.inputDocument).uriref()
good = 1
categories = kb.each(t, rdf.type)
for cat in categories:
if cat is triage.knownError:
if verbose: print "\tknown failure: "+ inputDocument[-40:]
good = 0
if good:
n3PositiveTestData.append((t.uriref(), case, description, inputDocument))
for t in kb.each(pred=rdf.type, obj=n3test.NegativeParserTest):
u = t.uriref()
hash = u.rfind("#")
slash = u.rfind("/")
assert hash >0 and slash > 0
case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
description = str(kb.the(t, n3test.description))
# if description == None: description = case + " (no description)"
inputDocument = kb.the(t, n3test.inputDocument).uriref()
n3NegativeTestData.append((t.uriref(), case, description, inputDocument))
for tt in kb.each(pred=rdf.type, obj=sparql_manifest.Manifest):
for t in kb.the(subj=tt, pred=sparql_manifest.entries):
name = str(kb.the(subj=t, pred=sparql_manifest.name))
query_node = kb.the(subj=t, pred=sparql_manifest.action)
if isinstance(query_node, AnonymousNode):
data = ''
for data_node in kb.each(subj=query_node, pred=sparql_query.data):
data = data + ' ' + data_node.uriref()
inputDocument = kb.the(subj=query_node, pred=sparql_query.query).uriref()
else:
data = ''
inputDocument = query_node.uriref()
j = inputDocument.rfind('/')
case = inputDocument[j+1:]
outputDocument = kb.the(subj=t, pred=sparql_manifest.result)
if outputDocument:
outputDocument = outputDocument.uriref()
else:
outputDocument = None
good = 1
status = kb.the(subj=t, pred=dawg_test.approval)
if status != dawg_test.Approved:
print status, name
if verbose: print "\tNot approved: "+ inputDocument[-40:]
good = 0
if good:
sparqlTestData.append((tt.uriref(), case, name, inputDocument, data, outputDocument))
for t in kb.each(pred=rdf.type, obj=test.PerformanceTest):
x = t.uriref()
theTime = kb.the(subj=t, pred=test.pyStones)
description = str(kb.the(t, test.description))
arguments = str(kb.the(t, test.arguments))
environment = kb.the(t, test.environment)
if environment == None: env=""
else: env = str(environment) + " "
perfData.append((x, theTime, description, env, arguments))
testData.sort()
cwmTests = len(testData)
if verbose: print "Cwm tests: %i" % cwmTests
RDFTestData.sort()
RDFNegativeTestData.sort()
rdfTests = len(RDFTestData)
rdfNegativeTests = len(RDFNegativeTestData)
perfData.sort()
perfTests = len(perfData)
n3PositiveTestData.sort()
n3PositiveTests = len(n3PositiveTestData)
n3NegativeTestData.sort()
n3NegativeTests = len(n3NegativeTestData)
sparqlTestData.sort()
sparqlTests = len(sparqlTestData)
totalTests = cwmTests + rdfTests + rdfNegativeTests + sparqlTests \
+ perfTests + n3PositiveTests + n3NegativeTests
if verbose: print "RDF parser tests: %i" % rdfTests
for t, u, case, refFile, description, env, arguments, verboseDebug in testData:
tests = tests + 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, description)
# print " %scwm %s giving %s" %(arguments, case)
assert case and description and arguments
cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e 's;%s;%s;g'""" % (WD, REFWD,
cwm_command, '../cwm.py')
if normal:
execute("""CWM_RUN_NS="run#" %s %s %s --quiet %s | %s > ,temp/%s""" %
(env, python_command, cwm_command, arguments, cleanup , case))
if diff(case, refFile):
problem("######### from normal case %s: %scwm %s" %( case, env, arguments))
continue
if chatty and not verboseDebug:
execute("""%s %s %s --chatty=100 %s &> /dev/null""" %
(env, python_command, cwm_command, arguments), noStdErr=True)
if proofs and kb.contains(subj=t, pred=rdf.type, obj=test.CwmProofTest):
execute("""%s %s %s --quiet %s --base=a --why > ,proofs/%s""" %
(env, python_command, cwm_command, arguments, case))
execute("""%s ../check.py < ,proofs/%s | %s > ,temp/%s""" %
(python_command, case, cleanup , case))
if diff(case, refFile):
problem("######### from proof case %s: %scwm %s" %( case, env, arguments))
# else:
# progress("No proof for "+`t`+ " "+`proofs`)
# progress("@@ %s" %(kb.each(t,rdf.type)))
passes = passes + 1
for u, case, name, inputDocument, data, outputDocument in sparqlTestData:
tests += 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, name)
inNtriples = case + '_1'
outNtriples = case + '_2'
try:
execute("""%s %s %s --sparql=%s --filter=%s --filter=%s --ntriples > ',temp/%s'""" %
(python_command, cwm_command, data, inputDocument,
'sparql/filter1.n3', 'sparql/filter2.n3', inNtriples))
except NotImplementedError:
pass
except:
problem(str(sys.exc_info()[1]))
if outputDocument:
execute("""%s %s %s --ntriples > ',temp/%s'""" %
(python_command, cwm_command, outputDocument, outNtriples))
if rdfcompare3(inNtriples, ',temp/' + outNtriples):
problem('We have a problem with %s on %s' % (inputDocument, data))
passes += 1
for u, case, description, inputDocument, outputDocument in RDFTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument and outputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
execute("""%s %s --quiet --rdf=RT %s --ntriples > ,temp/%s""" %
(python_command, cwm_command, inputDocument, case))
if rdfcompare3(case, localize(outputDocument)):
problem(" from positive parser test %s running\n\tcwm %s\n" %( case, inputDocument))
passes = passes + 1
for u, case, description, inputDocument in RDFNegativeTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s --quiet --rdf=RT %s --ntriples > ,temp/%s 2>/dev/null""" %
(python_command, cwm_command, inputDocument, case))
except:
pass
else:
problem("""I didn't get a parse error running python %s --quiet --rdf=RT %s --ntriples > ,temp/%s
from test ^=%s
I should have.
""" %
(cwm_command, inputDocument, case, u))
passes = passes + 1
for u, case, description, inputDocument in n3PositiveTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s 2>/dev/null""" %
(python_command, '../grammar/predictiveParser.py', inputDocument, case))
except RuntimeError:
problem("""Error running ``python %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s 2>/dev/null''""" %
('../grammar/predictiveParser.py', inputDocument, case))
passes = passes + 1
for u, case, description, inputDocument in n3NegativeTestData:
tests = tests + 1
if tests < start: continue
print "%3i/%i) %s %s" %(tests, totalTests, case, description)
# print " %scwm %s giving %s" %(inputDocument, case)
assert case and description and inputDocument
# cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
# WD, REFWD)
try:
execute("""%s %s ../grammar/n3-selectors.n3 http://www.w3.org/2000/10/swap/grammar/n3#document %s > ,temp/%s 2>/dev/null""" %
(python_command, '../grammar/predictiveParser.py', inputDocument, case))
except:
pass
else:
problem("""There was no error executing ``python %s --grammar=../grammar/n3-selectors.n3 --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s > ,temp/%s''
There should have been one.""" %
('../grammar/predictiveParser.py', inputDocument, case))
passes = passes + 1
timeMatcher = re.compile(r'\t([0-9]+)m([0-9]+)\.([0-9]+)s')
## from test.pystone import pystones
## pyStoneTime = pystones()[1]
for u, theTime, description, env, arguments in perfData:
tests = tests + 1
if tests < start: continue
urel = refTo(base(), u)
print "%3i/%i %-30s %s" %(tests, totalTests, urel, description)
tt = os.times()[-1]
a = system("""%s %s %s --quiet %s >,time.out""" %
(env, python_command, cwm_command, arguments))
userTime = os.times()[-1] - tt
print """%spython %s --quiet %s 2>,time.out""" % \
(env, cwm_command, arguments)
## c = file(',time.out', 'r')
## timeOutput = c.read()
## c.close()
## timeList = [timeMatcher.search(b).groups() for b in timeOutput.split('\n') if timeMatcher.search(b) is not None]
## print timeList
## userTimeStr = timeList[1]
## userTime = int(userTimeStr[0])*60 + float(userTimeStr[1] + '.' + userTimeStr[2])
pyCount = pyStoneTime * userTime
print pyCount
if problems != []:
sys.stderr.write("\nProblems:\n")
for s in problems:
sys.stderr.write(" " + s + "\n")
raise RuntimeError("Total %i errors in %i tests." % (len(problems), tests))
def figureBalances(startDate, endDate, inputURIs=["/dev/stdin"]):
global verbose
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# Load the data:
kb = loadMany(inputURIs);
rates = loadMany(["currencies.n3"]);
sts = kb.statementsMatching(pred = OFX.BANKTRANLIST);
#if verbose:
# print len(sts), " bank transaction lists."
lists = []
for st in sts:
tl = st.object();
start = str(kb.any(tl, OFX.DTSTART))[:10];
end = str(kb.any(tl, OFX.DTEND))[:10];
# print "Transaction list %s - %s " %(start, end)
lists.append((start, end, tl, st.subject()))
lists.sort(reverse = 1);
# lists.reverse();
balances = [];
first = {};
g = {};
for s, e, t, stmtrs in lists:
# Do one statement, working backward to get the balance each date
ac = kb.any(stmtrs, OFX.BANKACCTFROM);
if ac == None: ac = kb.the(stmtrs, OFX.CCACCTFROM);
#info("ac = "+`ac`)
acid = str(kb.the(ac, OFX.ACCTID))[-4:];
# info("Bank statment %s to %s for %s" % (s, e, acid)); # @@
ledgerBalance = kb.the(stmtrs, OFX.LEDGERBAL);
curdef = kb.the(stmtrs, OFX.CURDEF).value();
currency = cur.sym(curdef);
conversionRate = 1
if (curdef != "USD"):
conversionRate = rates.the(currency, cur.in_USD).value(); # , None, kb.store.symbol(currencySource)).value();
balanceDate = str(kb.the(ledgerBalance, OFX.DTASOF))[:10];
balance = float(str(kb.the(ledgerBalance, OFX.BALAMT))) * conversionRate;
transactionsThisStatement = [];
for tran in kb.each(t, OFX.STMTTRN):
transactionsThisStatement.append((
str(kb.the(tran, OFX.DTPOSTED))[:10],
conversionRate * float(str(str(kb.the(tran, OFX.TRNAMT))))));
transactionsThisStatement.sort();
transactionsThisStatement.reverse();
bal, dat = float(str(balance)), balanceDate;
for d, a in transactionsThisStatement:
# assert dat >= d, "Ooops '%s' < '%s' %d, %d in %s" % (dat, d, len(dat), len(d), acid)
# print "\t\t%10s %10s\t%s\t%10.2f\t%10.2f" % (d, dat, acid, a, bal)
balances.append((d, dat, acid, bal));
bal = bal - a
first[acid] = [d, bal];
dat = d
balances.sort();
if verbose: info("First: " + `first`)
return first, balances
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
"""Fin - financial summary
<command> <options> <inputURIs>
Totals transactions by classes to which they are known to belong
This is or was http://www.w3.org/2000/10/swap/pim/fin.py
"""
#import urllib
#import time
import sys
# global sax2rdf
global kb, tax
def noteError(e):
if not errors.get(s, None): errors[s] = [];
errors[s].append(e)
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
fatalErrors = 0;
# Load the data:
kb = loadMany(inputURIs)
qu_date = qu.date
qu_in_USD = qu.in_USD
qu_amount = qu.amount
qu_payee = qu.payee
qu_Classified = qu.Classified
qu_Unclassified = qu.Unclassified
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
if verbose:
progress("Tax categories" + `taxCategories`)
specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]
####### Analyse the data:
numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
monthTotals = [0] * numberOfMonths
incomeByMonth = [0] * numberOfMonths
income, outgoings = 0,0
outgoingsByMonth = [0] * numberOfMonths
quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
bottomCategories = [];
for c in quCategories:
if isBottomClass(c): bottomCategories.append(c);
totals = {} # Total by all classes of transaction
count = {} # Number of transactions
byMonth = {}
sts = kb.statementsMatching(pred=qu.amount) # Ideally one per transaction
errors = {}
for st in sts:
s = st.subject()
uri = s.uriref()
# classified = kb.each(pred=rdf_type, obj=qu_Classified)
# unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
# for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
# for s in classified + unclassified:
# progress( "Transaction ", `s`)
t_ok, c_ok = 0, 0
cats = allClasses(kb.each(subj=s, pred=rdf.type))
# progress( "Categories: "+`cats`)
month = monthNumber(s)
if month not in range(numberOfMonths) : continue
payees = kb.each(subj=s, pred=qu_payee)
if not payees:
progress("@@@ Error: No payee for "+`uri`)
payee = "@@@@@@ Unknown";
fatalErrors += 1;
elif len(payees) >1 and str(payees[0]) == "Check":
payee = payees[1]
else:
payee = payees[0]
amounts = kb.each(subj=s, pred=qu_in_USD)
if len(amounts) == 0:
amounts = kb.each(subj=s, pred=qu_amount)
if len(amounts) == 0:
progress("@@@ Error: No USD amount for "+`uri`)
fatalErrors += 1;
else:
progress("Warning: No USD amount for "+`uri`+", assuming USD")
if len(amounts) >1:
if (cat_ns.Internal not in cats or
len(amounts) != 2 ):
fatalErrors += 1;
progress(
"Error: More than one USD amount %s for transaction %s -- ignoring!\n"
% (`amounts`,uri))
else:
sum = float(amounts[0]) + float(amounts[1])
if sum != 0:
fatalErrors += 1;
progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
% (sum, amounts, uri))
continue
if len(amounts) != 1:
progress("@@@ Error: No amount for "+`uri`);
fatalErrors += 1;
ss = kb.statementsMatching(subj=s)
progress(`ss`+'; KB='+`kb.n3String()`)
continue
amount = float(amounts[0].__str__())
# print "%s %40s %10s month %i" %(date, payee, `amount`, month)
monthTotals[month] = monthTotals[month] + amount
if cat_ns.Internal not in cats:
if amount > 0:
incomeByMonth[month] = incomeByMonth[month] + amount
income = income + amount
else:
outgoingsByMonth[month] = outgoingsByMonth[month] + amount
outgoings = outgoings + amount
normalCats = [] # For this item
for c in cats:
totals[c] = totals.get(c, 0) + amount
byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
count[c] = count.get(c, 0) + 1
byMonth[c][month] = byMonth[c][month] + amount
if c not in specialCategories:
normalCats.append(c)
bottomCats = normalCats[:] # Copy
for b in normalCats:
sups = kb.each(subj=b, pred=rdfs.subClassOf)
for sup in sups:
if sup in bottomCats:
bottomCats.remove(sup)
if len(bottomCats) == 0:
noteError("No categoriy: %s for <%s>" # all cats: %s, raw cats:%s"
%(`cats`, `s`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
noteError("Be more specifc: %s for <%s>" %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
if len(bottomCats) > 1:
noteError("Inconsistent categories: %s" # all cats: %s, raw cats:%s"
%(`bottomCats`)) # ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
print '<html xmlns="http://www.w3.org/1999/xhtml">'
if '--summary' in sys.argv:
title = "Monthly summary"
elif '--issues' in sys.argv:
title = "Issues"
else:
title = "Report"
print """<head>
<meta charset='UTF-8'>
<title>%s</title>
<link rel="Stylesheet" href="report.css">
</head>
<body>
""" % (title)
# <img src="sand-dollar.gif" alt="dollar" align="right"/>
version = "$Id: fin.py,v 1.31 2014-07-23 13:23:05 timbl Exp $"
# SUMMARY TABLE OF CATEGORY BY MONTH
if '--summary' in sys.argv:
print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
print "<tr><th></th><th>Total </th>"
for month in range(numberOfMonths):
m = month + int(startDate[5:7]) - 1
while m > 11: m -= 12 # Modulo in python?
print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
print "</tr>"
def listFor(c, depth=0): # Any, because there could be 2 copies of same list :-(
subs = kb.any(subj = c, pred = owl.disjointUnionOf);
res = [ (c, depth) ];
if subs == None:
subs = kb.each(pred = rdfs.subClassOf, obj = c);
if len(subs) > 0:
sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
for sub in subs: res += listFor(sub, depth+1)
else:
for sub in subs: res += listFor(sub, depth+1)
return res
printOrder = listFor(qu.Transaction);
for cat, depth in printOrder:
label = kb.the(subj=cat, pred=rdfs.label)
if label == None:
label = `cat`
sys.stderr.write("@@ No label for "+`cat` +"\n")
else:
label = str(label)
anchor = cat.fragid
if totals.get(cat, None) != None:
print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
numberOfMonths, indent = depth)
print "<tr><td colspan='14'> ___ </td></tr>"
print monthGridRow("Income", None, income, incomeByMonth, numberOfMonths)
print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)
print "</table>"
# Chart of income stacked up against expenses
if '--charts' in sys.argv:
print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"
writeChart(filename = "chart.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 1)
writeChart(filename = "all.svg",
categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
totals = totals, income=income, outgoings=outgoings, shortTerm = 0)
# Output totals
if (totalsFilename):
ko = kb.newFormula()
for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
ko.close()
fo = open(totalsFilename, "w")
fo.write(ko.n3String())
fo.close
if '--issues' in sys.argv:
# Generate a list of errors found
errstr = ""
for x, list in errors.items():
errstr += transactionRow(x)
for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n" # @@@ encode error string
if errstr:
print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
# List Unclassified Income and Spending
def transactionList(cat):
ts = kb.each(pred = rdf.type, obj = cat)
if len(ts) == 0: return ""
label = kb.any(cat, rdfs.label)
st = '<h2>'+label.value()+'</h2>\n'
return st + transactionTable(ts)
for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
print transactionList(cat)
print reimbursablesCheck();
internalCheck()
if 0:
print "<h2>Tax Categories</h2>"
taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)
print "<h2>Tax stuff</h2>"
print "<table>"
print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
print "</table>"
# print "<h2>Personal Category total</h2>"
# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)
print
print "Note totals for tax and personal breakdowns must match."
dates = kb.statementsMatching(pred=qu.date)
print "There should be a total of %i transactions in each." % len(dates)
if 0:
print "<pre>(consistency check)"
problems = 0
for s in dates:
tra = s.subject()
types = kb.each(subj=tra, pred=rdf_type)
for typ in types:
if typ is qu.Unclassified or typ is qu.Classified:
break # ok
else:
print "@@@@ problem transcation with no classified or unclassified, with types", types
printTransactionDetails(tra)
problems = problems + 1
print problems, "problems.</pre>"
print "</body></html>"
return fatalErrors