Example #1
0
def loadFiles(files):
    graph = myStore.formula()
    graph.setClosureMode("e")    # Implement sameAs by smushing
    if verbose>0: progress("Loading %s..." % files)
    graph = myStore.loadMany(files, openFormula=graph)
    if verbose>0: progress("Loaded", graph)
    return graph
Example #2
0
def main():
    """The main function


    """ ### """
    try:
        opts, testFiles = getopt.getopt(sys.argv[1:], "hc:o:e:",
            ["help", "command=", "output=", "error="])
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)
    errorFile = ',temp/__error.txt'   # was "/dev/null"
    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        if o in ("-c", "--command"):
            commandFile = a
        if o in ("-o", "--output"):
            outputFile = a
        if o in ("-e", "--error"):
            errorFile = a

    assert system("mkdir -p ,temp") == 0
    assert system("mkdir -p ,diffs") == 0

    testFiles = testFiles + [
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql1/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql2/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql3/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql4/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bnode-coreference/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/boolean-effective-value/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/construct/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/regex/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/sort/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/triple-match/manifest.ttl',
        'http://www.w3.org/2001/sw/DataAccess/tests/data-r2/type-promotion/manifest.ttl']
    kb = loadMany(testFiles, referer="")
    output = formula()
    testCwmSparql(kb, output, errorFile)
    output.close()
    output.store.dumpNested(output, ToN3(file(outputFile, 'w').write))
Example #3
0
def lookUp(predicates, assumptions=Set()):
    """Look up all the schemas for the predicates given"""
    global verbose
    schemas = assumptions
    for pred in predicates:
        if verbose > 3: progress("Predicate: %s" % `pred`)
        u = pred.uriref()
        hash = u.find("#")
        if hash <0:
            if verbose > 1: progress("Warning: Predicate <%s> looks like web resource not Property" % u)
        else:
            schemas.add(u[:hash])
    if verbose > 2:
        for r in schemas:
            progress("Metadata to be loaded: ", r) 
    if schemas:
        return loadMany([(x) for x in schemas])
    return myStore.store.newFormula() # Empty formula
Example #4
0
def main():
    """The main function


    """ ### """
    try:
        opts, testFiles = getopt.getopt(sys.argv[1:], "hc:o:e:",
            ["help", "command=", "output=", "error="])
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)
    errorFile = ',temp/__error.txt'   # was "/dev/null"
    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        if o in ("-c", "--command"):
            commandFile = a
        if o in ("-o", "--output"):
            outputFile = a
        if o in ("-e", "--error"):
            errorFile = a
    commands = [w[:-1] for w in file(commandFile, 'r')]

    assert system("mkdir -p ,temp") == 0
    assert system("mkdir -p ,diffs") == 0

    kb = loadMany(testFiles, referer="")
    z = 0
    for command in commands:
        output = formula()
        testParser(command, kb, output, errorFile)

        output.close()
        output.store.dumpNested(output, ToN3(file(outputFile+`z`, 'w').write))
        z = z+1
Example #5
0
def loadFiles(files):
    graph = myStore.formula()
    graph.setClosureMode("e")    # Implement sameAs by smushing
    graph = myStore.loadMany(files, openFormula=graph)
    if verbose: progress("Loaded", graph, graph.__class__)
    return graph
Example #6
0
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
    """Fin - financial summary
        
 <command> <options> <inputURIs>
 Totals transactions by classes to which they are known to belong 
 This is or was  http://www.w3.org/2000/10/swap/pim/fin.py
 
"""
        
    #import urllib
    #import time
    import sys
    # global sax2rdf
    global kb, tax
    
    def noteError(e):
        if not errors.get(s, None): errors[s] = [];
        errors[s].append(e)
    
    # The base URI for this process - the Web equiv of cwd
    _baseURI = uripath.base()
    
    _outURI = _baseURI
    option_baseURI = _baseURI   # To start with - then tracks running base
    fatalErrors = 0;

# Load the data:

    kb = loadMany(inputURIs)
            
    qu_date = qu.date
    qu_in_USD = qu.in_USD
    qu_amount = qu.amount
    qu_payee = qu.payee
    qu_Classified = qu.Classified
    qu_Unclassified = qu.Unclassified
    taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
    if verbose:
        progress("Tax categories" + `taxCategories`)
    specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]

####### Analyse the data:
    numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
    monthTotals = [0] * numberOfMonths
    incomeByMonth = [0] * numberOfMonths
    income, outgoings = 0,0
    outgoingsByMonth = [0] * numberOfMonths

    quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
    bottomCategories = [];
    for c in quCategories:
        if isBottomClass(c): bottomCategories.append(c);
    
    totals = {}  # Total by all classes of transaction
    count = {}  # Number of transactions
    byMonth = {}

    sts = kb.statementsMatching(pred=qu.amount)  # Ideally one per transaction
    errors = {}
    for st in sts:
        s = st.subject()
        uri = s.uriref()
#        classified =  kb.each(pred=rdf_type, obj=qu_Classified)
#        unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
#        for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
#        for s in classified + unclassified:
#           progress( "Transaction ", `s`)
        t_ok, c_ok = 0, 0
        cats = allClasses(kb.each(subj=s, pred=rdf.type))
        # progress( "Categories: "+`cats`)
        
        month = monthNumber(s)
        if month not in range(numberOfMonths) : continue
                    
        payees = kb.each(subj=s, pred=qu_payee)
	if not payees:
	    progress("@@@ Error: No payee for "+`uri`)
	    payee = "@@@@@@ Unknown";
            fatalErrors += 1;
	
        elif len(payees) >1 and str(payees[0]) == "Check":
	    payee = payees[1]
        else:
	    payee = payees[0]
        
        amounts = kb.each(subj=s, pred=qu_in_USD)
        if len(amounts) == 0:
            amounts = kb.each(subj=s, pred=qu_amount)
            if len(amounts) == 0:
                progress("@@@ Error: No USD amount for "+`uri`)
                fatalErrors += 1;

            else:
                progress("Warning: No USD amount for "+`uri`+", assuming USD")
        if len(amounts) >1:
            if (cat_ns.Internal not in cats or
                len(amounts) != 2 ):
                fatalErrors += 1;

                progress(
        "Error: More than one USD amount %s for transaction %s -- ignoring!\n"
                        % (`amounts`,uri))
            else:
                sum = float(amounts[0]) + float(amounts[1])
                if sum != 0:
                    fatalErrors += 1;
                    progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
                        % (sum, amounts, uri))
            continue

        if len(amounts) != 1:
            progress("@@@ Error: No amount for "+`uri`);
            fatalErrors += 1;
            ss = kb.statementsMatching(subj=s)
            progress(`ss`+'; KB='+`kb.n3String()`)
            continue
        amount = float(amounts[0].__str__())
#           print "%s  %40s  %10s month %i" %(date, payee, `amount`, month)

        monthTotals[month] = monthTotals[month] + amount
        if cat_ns.Internal not in cats:
            if amount > 0:
                incomeByMonth[month] = incomeByMonth[month] + amount
                income = income + amount
            else:
                outgoingsByMonth[month] = outgoingsByMonth[month] + amount
                outgoings = outgoings + amount

        normalCats = []  # For this item
        for c in cats:
            totals[c] = totals.get(c, 0) + amount
            byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
            count[c] = count.get(c, 0) + 1
            byMonth[c][month] = byMonth[c][month] + amount
            if c not in specialCategories:
                normalCats.append(c)
        bottomCats = normalCats[:] # Copy
        for b in normalCats:
            sups = kb.each(subj=b, pred=rdfs.subClassOf)
            for sup in sups:
                if sup in bottomCats:
                    bottomCats.remove(sup)
        if len(bottomCats) == 0:
           noteError("No categoriy: %s  for <%s>"  # all cats: %s, raw cats:%s"
                        %(`cats`, `s`))  #  ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
        elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
           noteError("Be more specifc: %s for <%s>"  %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
        if len(bottomCats) > 1:
           noteError("Inconsistent categories: %s"  # all cats: %s, raw cats:%s"
                        %(`bottomCats`))  #  ,`cats`, `kb.each(subj=s, pred=rdf.type)`)

    
    print '<html xmlns="http://www.w3.org/1999/xhtml">'

    if '--summary' in sys.argv:
        title = "Monthly summary"
    elif '--issues' in sys.argv:
        title = "Issues"
    else:
        title = "Report"

    print """<head>
        <meta charset='UTF-8'>
        <title>%s</title>
        <link rel="Stylesheet" href="report.css">
    </head>
    <body>
    """ % (title)
#           <img src="sand-dollar.gif" alt="dollar" align="right"/>
    


    version = "$Id$"


#  SUMMARY  TABLE OF CATEGORY BY MONTH

    if '--summary' in sys.argv:
	print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
	print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
	
	print "<tr><th></th><th>Total </th>" 
	for month in range(numberOfMonths):
	    m = month + int(startDate[5:7]) - 1
	    while m > 11: m -= 12  # Modulo in python?
	    
	    
	    print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
	print "</tr>"
	
	
	def listFor(c, depth=0):   # Any, because there could be 2 copies of same list :-(
	    subs = kb.any(subj = c, pred = owl.disjointUnionOf);
	    res = [ (c, depth) ];
	    if subs == None:
		subs = kb.each(pred = rdfs.subClassOf, obj = c);
		if len(subs) > 0:
		    sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
		for sub in subs: res += listFor(sub, depth+1)
	    else:
		for sub in subs: res += listFor(sub, depth+1)
	    return res
	    
	printOrder = listFor(qu.Transaction);
	
	for cat, depth in printOrder:
	    label = kb.the(subj=cat, pred=rdfs.label)
	    if label == None:
		label = `cat`
		sys.stderr.write("@@ No label for "+`cat` +"\n")
	    else:
		label = str(label)
	    anchor = cat.fragid
	    if totals.get(cat, None) != None:
		print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
		    numberOfMonths, indent = depth)

	print "<tr><td colspan='14'> ___  </td></tr>"
	print monthGridRow("Income", None,  income, incomeByMonth, numberOfMonths)
	print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
	print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)

	print "</table>"

    
#  Chart of income stacked up against expenses
    if '--charts' in sys.argv:
	print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
	print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"

	writeChart(filename = "chart.svg",
	    categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
	    totals = totals, income=income, outgoings=outgoings, shortTerm = 1)

	writeChart(filename = "all.svg",
	    categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
        totals = totals, income=income, outgoings=outgoings, shortTerm = 0)


    # Output totals
    
    
    if (totalsFilename):
    
	ko = kb.newFormula()    
	for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
	    ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
	ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
	ko.close()
        fo = open(totalsFilename, "w")
        fo.write(ko.n3String())
        fo.close

    if '--issues' in sys.argv:

	#  Generate a list of errors found
	errstr = ""
	for x, list in errors.items():
	    errstr += transactionRow(x)
	    for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n"  #  @@@ encode error string
	if errstr:
	    print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
	
	# List Unclassified Income and Spending
	
	def transactionList(cat):
	    ts = kb.each(pred = rdf.type, obj = cat)
	    if len(ts) == 0: return ""
	    label = kb.any(cat, rdfs.label)
	    st = '<h2>'+label.value()+'</h2>\n'
	    return st + transactionTable(ts)
	
	for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
	    print transactionList(cat)

	print reimbursablesCheck();
	
	internalCheck()

    if 0:
	print "<h2>Tax Categories</h2>"
	taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
	printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)

	print "<h2>Tax stuff</h2>"
	print "<table>"
	print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
	print "</table>"
        
	# print "<h2>Personal Category total</h2>"
	# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)

	print

	print "Note totals for tax and personal breakdowns must match."
	dates = kb.statementsMatching(pred=qu.date)
	print "There should be a total of %i transactions in each." % len(dates)

    

    if 0:
        print "<pre>(consistency check)"
        problems = 0
        for s in dates:
            tra = s.subject()
            types = kb.each(subj=tra, pred=rdf_type)
            for typ in types:
                if typ is qu.Unclassified or typ is qu.Classified:
                    break # ok
            else:
                print "@@@@ problem transcation with no classified or unclassified, with types", types
                printTransactionDetails(tra)
                problems = problems + 1
        print problems, "problems.</pre>"
    
    print "</body></html>"
    return fatalErrors
Example #7
0
def main():
    global verbose, proofs, chatty, normal, no_action
    start = 1
    cwm_command='../cwm.py'
    python_command='python -tt'
    global ploughOn # even if error
    ploughOn = 0
    global verbose
    verbose = 0
    global just_fix_it
    just_fix_it = 0
    if diag.print_all_file_names:
        a = file('testfilelist','w')
        a.write('')
        a.close()
    try:
        opts, testFiles = getopt.getopt(sys.argv[1:], "h?s:nNcipf:v",
            ["help", "start=", "testsFrom=", "no-action", "No-normal", "chatty",
                "ignoreErrors", "proofs", "verbose","overwrite","cwm="])
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)
    output = None
    for o, a in opts:
        if o in ("-h", "-?", "--help"):
            usage()
            sys.exit()
        if o in ("-v", "--verbose"):
            verbose = 1
        if o in ("-i", "--ignoreErrors"):
            ploughOn = 1
        if o in ("-s", "--start"):
            start = int(a)
        if o in ("-f", "--testsFrom"):
            testFiles.append(a)
        if o in ("-n", "--no-action"):
            no_action = 1
        if o in ("-N", "--No-normal"):
            normal = 0
        if o in ("-c", "--chatty"):
            chatty = 1
        if o in ("-p", "--proofs"):
            proofs = 1
        if o in ("--overwrite",):
            just_fix_it = 1
        if o in ("--cwm", "--the_end"):
            cwm_command=a

    
    assert system("mkdir -p ,temp") == 0
    assert system("mkdir -p ,diffs") == 0
    if proofs: assert system("mkdir -p ,proofs") == 0
    
    tests=0
    passes=0
    global problems
    problems = []
    
    REFWD="http://example.com/swap/test"
    WD = base()[:-1] 
    
    #def basicTest(case, desc, args)

    if verbose: progress("Test files:", testFiles)
    
    kb = loadMany(testFiles, referer="")
    testData = []
    RDFTestData  = []
    RDFNegativeTestData = []
    perfData = []
    n3PositiveTestData = []
    n3NegativeTestData = []
    sparqlTestData = []
#    for fn in testFiles:
#       print "Loading tests from", fn
#       kb=load(fn)
    
    for t in kb.each(pred=rdf.type, obj=test.CwmTest):
        verboseDebug = kb.contains(subj=t, pred=rdf.type, obj=test.VerboseTest)
        u = t.uriref()
        ref = kb.the(t, test.referenceOutput)
        if ref == None:
            case = str(kb.the(t, test.shortFileName))
            refFile = "ref/%s" % case
        else:
            refFile = refTo(base(), ref.uriref())
            case  = ""
            for ch in refFile:
                if ch in "/#": case += "_"
                else: case += ch  # Make up test-unique temp filename
        description = str(kb.the(t, test.description))
        arguments = str(kb.the(t, test.arguments))
        environment = kb.the(t, test.environment)
        if environment == None: env=""
        else: env = str(environment) + " "
        testData.append((t, t.uriref(), case, refFile, description, env, arguments, verboseDebug))

    for t in kb.each(pred=rdf.type, obj=rdft.PositiveParserTest):

        x = t.uriref()
        y = x.find("/rdf-tests/")
        x = x[y+11:] # rest
        for i in range(len(x)):
            if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
        case = "rdft_" + x + ".nt" # Hack - temp file name
        
        description = str(kb.the(t, rdft.description))
#           if description == None: description = case + " (no description)"
        inputDocument = kb.the(t, rdft.inputDocument).uriref()
        outputDocument = kb.the(t, rdft.outputDocument).uriref()
        status = kb.the(t, rdft.status).string
        good = 1
        if status != "APPROVED":
            if verbose: print "\tNot approved: "+ inputDocument[-40:]
            good = 0
        categories = kb.each(t, rdf.type)
        for cat in categories:
            if cat is triage.ReificationTest:
                if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
                good = 0
##            if cat is triage.ParseTypeLiteralTest:
##                if verbose: print "\tNot supported (Parse type literal): "+ inputDocument[-40:]
##                good = 0
        if good:
            RDFTestData.append((t.uriref(), case, description,  inputDocument, outputDocument))

    for t in kb.each(pred=rdf.type, obj=rdft.NegativeParserTest):

        x = t.uriref()
        y = x.find("/rdf-tests/")
        x = x[y+11:] # rest
        for i in range(len(x)):
            if x[i]in"/#": x = x[:i]+"_"+x[i+1:]
        case = "rdft_" + x + ".nt" # Hack - temp file name
        
        description = str(kb.the(t, rdft.description))
#           if description == None: description = case + " (no description)"
        inputDocument = kb.the(t, rdft.inputDocument).uriref()
        status = kb.the(t, rdft.status).string
        good = 1
        if status != "APPROVED":
            if verbose: print "\tNot approved: "+ inputDocument[-40:]
            good = 0
        categories = kb.each(t, rdf.type)
        for cat in categories:
            if cat is triage.knownError:
                if verbose: print "\tknown failure: "+ inputDocument[-40:]
                good = 0
            if cat is triage.ReificationTest:
                if verbose: print "\tNot supported (reification): "+ inputDocument[-40:]
                good = 0
        if good:
            RDFNegativeTestData.append((t.uriref(), case, description,  inputDocument))



    for t in kb.each(pred=rdf.type, obj=n3test.PositiveParserTest):
        u = t.uriref()
        hash = u.rfind("#")
        slash = u.rfind("/")
        assert hash >0 and slash > 0
        case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
        
        description = str(kb.the(t, n3test.description))
#           if description == None: description = case + " (no description)"
        inputDocument = kb.the(t, n3test.inputDocument).uriref()
        good = 1
        categories = kb.each(t, rdf.type)
        for cat in categories:
            if cat is triage.knownError:
                if verbose: print "\tknown failure: "+ inputDocument[-40:]
                good = 0
        if good:
            n3PositiveTestData.append((t.uriref(), case, description,  inputDocument))


    for t in kb.each(pred=rdf.type, obj=n3test.NegativeParserTest):
        u = t.uriref()
        hash = u.rfind("#")
        slash = u.rfind("/")
        assert hash >0 and slash > 0
        case = u[slash+1:hash] + "_" + u[hash+1:] + ".out" # Make up temp filename
        
        description = str(kb.the(t, n3test.description))
#           if description == None: description = case + " (no description)"
        inputDocument = kb.the(t, n3test.inputDocument).uriref()

        n3NegativeTestData.append((t.uriref(), case, description,  inputDocument))

    for tt in kb.each(pred=rdf.type, obj=sparql_manifest.Manifest):
        for t in kb.the(subj=tt, pred=sparql_manifest.entries):
            name = str(kb.the(subj=t, pred=sparql_manifest.name))
            query_node = kb.the(subj=t, pred=sparql_manifest.action)
            if isinstance(query_node, AnonymousNode):
                data = ''
                for data_node in kb.each(subj=query_node, pred=sparql_query.data):
                    data = data + ' ' + data_node.uriref()

                inputDocument = kb.the(subj=query_node, pred=sparql_query.query).uriref()
            else:
                data = ''
                inputDocument = query_node.uriref()
            j = inputDocument.rfind('/')
            case = inputDocument[j+1:]
            outputDocument = kb.the(subj=t, pred=sparql_manifest.result)
            if outputDocument:
                outputDocument = outputDocument.uriref()
            else:
                outputDocument = None
            good = 1
            status = kb.the(subj=t, pred=dawg_test.approval)
            if status != dawg_test.Approved:
                print status, name
                if verbose: print "\tNot approved: "+ inputDocument[-40:]
                good = 0
            if good:
                sparqlTestData.append((tt.uriref(), case, name, inputDocument, data, outputDocument))
        



    for t in kb.each(pred=rdf.type, obj=test.PerformanceTest):
        x = t.uriref()
        theTime = kb.the(subj=t, pred=test.pyStones)
        description = str(kb.the(t, test.description))
        arguments = str(kb.the(t, test.arguments))
        environment = kb.the(t, test.environment)
        if environment == None: env=""
        else: env = str(environment) + " "
        perfData.append((x, theTime, description, env, arguments))

    testData.sort()
    cwmTests = len(testData)
    if verbose: print "Cwm tests: %i" % cwmTests
    RDFTestData.sort()
    RDFNegativeTestData.sort()
    rdfTests = len(RDFTestData)
    rdfNegativeTests = len(RDFNegativeTestData)
    perfData.sort()
    perfTests = len(perfData)
    n3PositiveTestData.sort()
    n3PositiveTests = len(n3PositiveTestData)
    n3NegativeTestData.sort()
    n3NegativeTests = len(n3NegativeTestData)
    sparqlTestData.sort()
    sparqlTests = len(sparqlTestData)
    totalTests = cwmTests + rdfTests + rdfNegativeTests + sparqlTests \
                 + perfTests + n3PositiveTests + n3NegativeTests
    if verbose: print "RDF parser tests: %i" % rdfTests

    for t, u, case, refFile, description, env, arguments, verboseDebug in testData:
        tests = tests + 1
        if tests < start: continue
        
        urel = refTo(base(), u)
    
        print "%3i/%i %-30s  %s" %(tests, totalTests, urel, description)
    #    print "      %scwm %s   giving %s" %(arguments, case)
        assert case and description and arguments
        cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e 's;%s;%s;g'""" % (WD, REFWD,
                                                                                                      cwm_command, '../cwm.py')
        
        if normal:
            execute("""CWM_RUN_NS="run#" %s %s %s --quiet %s | %s > ,temp/%s""" %
                (env, python_command, cwm_command, arguments, cleanup , case))  
            if diff(case, refFile):
                problem("######### from normal case %s: %scwm %s" %( case, env, arguments))
                continue

        if chatty and not verboseDebug:
            execute("""%s %s %s --chatty=100  %s  &> /dev/null""" %
                (env, python_command, cwm_command, arguments), noStdErr=True)   

        if proofs and kb.contains(subj=t, pred=rdf.type, obj=test.CwmProofTest):
            execute("""%s %s %s --quiet %s --base=a --why  > ,proofs/%s""" %
                (env, python_command, cwm_command, arguments, case))
            execute("""%s ../check.py < ,proofs/%s | %s > ,temp/%s""" %
                (python_command, case, cleanup , case)) 
            if diff(case, refFile):
                problem("######### from proof case %s: %scwm %s" %( case, env, arguments))
#       else:
#           progress("No proof for "+`t`+ " "+`proofs`)
#           progress("@@ %s" %(kb.each(t,rdf.type)))
        passes = passes + 1


    for u, case, name, inputDocument, data, outputDocument in sparqlTestData:
        tests += 1
        if tests < start: continue

        urel = refTo(base(), u)
        print "%3i/%i %-30s  %s" %(tests, totalTests, urel, name)
        inNtriples = case + '_1'
        outNtriples = case + '_2'
        try:
            execute("""%s %s %s --sparql=%s --filter=%s --filter=%s --ntriples > ',temp/%s'""" %
                    (python_command, cwm_command, data, inputDocument,
                     'sparql/filter1.n3', 'sparql/filter2.n3', inNtriples))
        except NotImplementedError:
            pass
        except:
            problem(str(sys.exc_info()[1]))
        if outputDocument:
            execute("""%s %s %s --ntriples > ',temp/%s'""" %
                    (python_command, cwm_command, outputDocument, outNtriples))
            if rdfcompare3(inNtriples, ',temp/' + outNtriples):
                problem('We have a problem with %s on %s' %  (inputDocument, data))


        passes += 1
        
    for u, case, description,  inputDocument, outputDocument in RDFTestData:
        tests = tests + 1
        if tests < start: continue
    
    
        print "%3i/%i)  %s   %s" %(tests, totalTests, case, description)
    #    print "      %scwm %s   giving %s" %(inputDocument, case)
        assert case and description and inputDocument and outputDocument
#       cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
#                       WD, REFWD)
        execute("""%s %s --quiet --rdf=RT %s --ntriples  > ,temp/%s""" %
            (python_command, cwm_command, inputDocument, case))
        if rdfcompare3(case, localize(outputDocument)):
            problem("  from positive parser test %s running\n\tcwm %s\n" %( case,  inputDocument))

        passes = passes + 1

    for u, case, description,  inputDocument in RDFNegativeTestData:
        tests = tests + 1
        if tests < start: continue
    
    
        print "%3i/%i)  %s   %s" %(tests, totalTests, case, description)
    #    print "      %scwm %s   giving %s" %(inputDocument, case)
        assert case and description and inputDocument
#       cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
#                       WD, REFWD)
        try:
            execute("""%s %s --quiet --rdf=RT %s --ntriples  > ,temp/%s 2>/dev/null""" %
            (python_command, cwm_command, inputDocument, case))
        except:
            pass
        else:
            problem("""I didn't get a parse error running python %s --quiet --rdf=RT %s --ntriples  > ,temp/%s
from test ^=%s
I should have.
""" %
            (cwm_command, inputDocument, case, u))

        passes = passes + 1


    for u, case, description, inputDocument in n3PositiveTestData:
        tests = tests + 1
        if tests < start: continue
    
    
        print "%3i/%i)  %s   %s" %(tests, totalTests, case, description)
    #    print "      %scwm %s   giving %s" %(inputDocument, case)
        assert case and description and inputDocument
#       cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
#                       WD, REFWD)
        try:
            execute("""%s %s --grammar=../grammar/n3-selectors.n3  --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s  > ,temp/%s 2>/dev/null""" %
            (python_command, '../grammar/predictiveParser.py', inputDocument, case))
        except RuntimeError:
            problem("""Error running ``python %s --grammar=../grammar/n3-selectors.n3  --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s  > ,temp/%s 2>/dev/null''""" %
            ('../grammar/predictiveParser.py', inputDocument, case))
        passes = passes + 1

    for u, case, description, inputDocument in n3NegativeTestData:
        tests = tests + 1
        if tests < start: continue
    
    
        print "%3i/%i)  %s   %s" %(tests, totalTests, case, description)
    #    print "      %scwm %s   giving %s" %(inputDocument, case)
        assert case and description and inputDocument
#       cleanup = """sed -e 's/\$[I]d.*\$//g' -e "s;%s;%s;g" -e '/@prefix run/d' -e '/^#/d' -e '/^ *$/d'""" % (
#                       WD, REFWD)
        try:
            execute("""%s %s ../grammar/n3-selectors.n3  http://www.w3.org/2000/10/swap/grammar/n3#document %s  > ,temp/%s 2>/dev/null""" %
                (python_command, '../grammar/predictiveParser.py', inputDocument, case))
        except:
            pass
        else:
            problem("""There was no error executing ``python %s --grammar=../grammar/n3-selectors.n3  --as=http://www.w3.org/2000/10/swap/grammar/n3#document --parse=%s    > ,temp/%s''
            There should have been one.""" %
                ('../grammar/predictiveParser.py', inputDocument, case))

        passes = passes + 1


    timeMatcher = re.compile(r'\t([0-9]+)m([0-9]+)\.([0-9]+)s')
##    from test.pystone import pystones
##    pyStoneTime = pystones()[1]
    for u, theTime, description, env, arguments in perfData:
        tests = tests + 1
        if tests < start: continue
        
        urel = refTo(base(), u)
    
        print "%3i/%i %-30s  %s" %(tests, totalTests, urel, description)
        tt = os.times()[-1]
        a = system("""%s %s %s --quiet %s >,time.out""" %
                       (env, python_command, cwm_command, arguments))
        userTime = os.times()[-1] - tt
        print """%spython %s --quiet %s 2>,time.out""" % \
                       (env, cwm_command, arguments)
##        c = file(',time.out', 'r')
##        timeOutput = c.read()
##        c.close()
##        timeList = [timeMatcher.search(b).groups() for b in timeOutput.split('\n') if timeMatcher.search(b) is not None]
##        print timeList
##        userTimeStr = timeList[1]
##        userTime = int(userTimeStr[0])*60 + float(userTimeStr[1] + '.' + userTimeStr[2])
        pyCount = pyStoneTime * userTime
        print pyCount
        
    if problems != []:
        sys.stderr.write("\nProblems:\n")
        for s in problems:
            sys.stderr.write("  " + s + "\n")
        raise RuntimeError("Total %i errors in %i tests." % (len(problems), tests))
Example #8
0
def figureBalances(startDate, endDate, inputURIs=["/dev/stdin"]):
    global verbose
    
    # The base URI for this process - the Web equiv of cwd
    _baseURI = uripath.base()
    
    _outURI = _baseURI
    option_baseURI = _baseURI   # To start with - then tracks running base


# Load the data:

    kb = loadMany(inputURIs);
    
    rates = loadMany(["currencies.n3"]);

    sts = kb.statementsMatching(pred = OFX.BANKTRANLIST);
    #if verbose:
        # print len(sts), " bank transaction lists."
    lists = []
    for st in sts:
        tl = st.object();
        start = str(kb.any(tl, OFX.DTSTART))[:10];
        end = str(kb.any(tl, OFX.DTEND))[:10];
        # print "Transaction list %s - %s " %(start, end)
        
        lists.append((start, end, tl, st.subject()))
        
    lists.sort(reverse = 1);
    # lists.reverse();
    balances = [];
    first = {};
    g = {};

    for s, e, t, stmtrs in lists:
    
        #   Do one statement, working backward to get the balance each date
        ac = kb.any(stmtrs, OFX.BANKACCTFROM);
        if ac == None: ac = kb.the(stmtrs, OFX.CCACCTFROM);
        #info("ac = "+`ac`)
        acid = str(kb.the(ac, OFX.ACCTID))[-4:];

        # info("Bank statment %s  to  %s for %s" % (s, e, acid)); # @@
        
        ledgerBalance = kb.the(stmtrs, OFX.LEDGERBAL);
        curdef = kb.the(stmtrs, OFX.CURDEF).value();
        currency = cur.sym(curdef);
        conversionRate = 1
        if (curdef != "USD"):
            conversionRate = rates.the(currency, cur.in_USD).value();  # , None, kb.store.symbol(currencySource)).value();
        balanceDate = str(kb.the(ledgerBalance, OFX.DTASOF))[:10];
        balance = float(str(kb.the(ledgerBalance, OFX.BALAMT))) * conversionRate;
        
        transactionsThisStatement = [];
        for tran in kb.each(t, OFX.STMTTRN):
            transactionsThisStatement.append((
                str(kb.the(tran, OFX.DTPOSTED))[:10], 
                conversionRate * float(str(str(kb.the(tran, OFX.TRNAMT))))));
        transactionsThisStatement.sort();
        transactionsThisStatement.reverse();
        
        bal, dat = float(str(balance)), balanceDate;
        for d, a in transactionsThisStatement:
            # assert dat >= d, "Ooops '%s' < '%s'  %d, %d in %s" % (dat, d, len(dat), len(d), acid)
            # print "\t\t%10s  %10s\t%s\t%10.2f\t%10.2f" % (d, dat, acid, a, bal)
            balances.append((d, dat, acid, bal));
            bal = bal - a
            first[acid] = [d, bal];
            dat = d
            
    balances.sort();
    if verbose: info("First: " + `first`)
    return first, balances
Example #9
0
def doCommand(startDate, endDate, inputURIs=["/dev/stdin"],totalsFilename=None):
    """Fin - financial summary
        
 <command> <options> <inputURIs>
 Totals transactions by classes to which they are known to belong 
 This is or was  http://www.w3.org/2000/10/swap/pim/fin.py
 
"""
        
    #import urllib
    #import time
    import sys
    # global sax2rdf
    global kb, tax
    
    def noteError(e):
        if not errors.get(s, None): errors[s] = [];
        errors[s].append(e)
    
    # The base URI for this process - the Web equiv of cwd
    _baseURI = uripath.base()
    
    _outURI = _baseURI
    option_baseURI = _baseURI   # To start with - then tracks running base
    fatalErrors = 0;

# Load the data:

    kb = loadMany(inputURIs)
            
    qu_date = qu.date
    qu_in_USD = qu.in_USD
    qu_amount = qu.amount
    qu_payee = qu.payee
    qu_Classified = qu.Classified
    qu_Unclassified = qu.Unclassified
    taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
    if verbose:
        progress("Tax categories" + `taxCategories`)
    specialCategories = taxCategories + [qu.Classified, qu.Unclassified, qu.Transaction]

####### Analyse the data:
    numberOfMonths = monthOfDate(endDate) - monthOfDate(startDate)
    monthTotals = [0] * numberOfMonths
    incomeByMonth = [0] * numberOfMonths
    income, outgoings = 0,0
    outgoingsByMonth = [0] * numberOfMonths

    quCategories = kb.each(pred=rdf_type, obj=qu.Cat)
    bottomCategories = [];
    for c in quCategories:
        if isBottomClass(c): bottomCategories.append(c);
    
    totals = {}  # Total by all classes of transaction
    count = {}  # Number of transactions
    byMonth = {}

    sts = kb.statementsMatching(pred=qu.amount)  # Ideally one per transaction
    errors = {}
    for st in sts:
        s = st.subject()
        uri = s.uriref()
#        classified =  kb.each(pred=rdf_type, obj=qu_Classified)
#        unclassified = kb.each(pred=rdf_type, obj=qu_Unclassified)
#        for t in classified: assert t not in unclassified, "Can't be classified and unclassified!"+`t`
#        for s in classified + unclassified:
#           progress( "Transaction ", `s`)
        t_ok, c_ok = 0, 0
        cats = allClasses(kb.each(subj=s, pred=rdf.type))
        # progress( "Categories: "+`cats`)
        
        month = monthNumber(s)
        if month not in range(numberOfMonths) : continue
                    
        payees = kb.each(subj=s, pred=qu_payee)
	if not payees:
	    progress("@@@ Error: No payee for "+`uri`)
	    payee = "@@@@@@ Unknown";
            fatalErrors += 1;
	
        elif len(payees) >1 and str(payees[0]) == "Check":
	    payee = payees[1]
        else:
	    payee = payees[0]
        
        amounts = kb.each(subj=s, pred=qu_in_USD)
        if len(amounts) == 0:
            amounts = kb.each(subj=s, pred=qu_amount)
            if len(amounts) == 0:
                progress("@@@ Error: No USD amount for "+`uri`)
                fatalErrors += 1;

            else:
                progress("Warning: No USD amount for "+`uri`+", assuming USD")
        if len(amounts) >1:
            if (cat_ns.Internal not in cats or
                len(amounts) != 2 ):
                fatalErrors += 1;

                progress(
        "Error: More than one USD amount %s for transaction %s -- ignoring!\n"
                        % (`amounts`,uri))
            else:
                sum = float(amounts[0]) + float(amounts[1])
                if sum != 0:
                    fatalErrors += 1;
                    progress("Sum %f not zero for USD amounts %s for internal transaction %s.\n"
                        % (sum, amounts, uri))
            continue

        if len(amounts) != 1:
            progress("@@@ Error: No amount for "+`uri`);
            fatalErrors += 1;
            ss = kb.statementsMatching(subj=s)
            progress(`ss`+'; KB='+`kb.n3String()`)
            continue
        amount = float(amounts[0].__str__())
#           print "%s  %40s  %10s month %i" %(date, payee, `amount`, month)

        monthTotals[month] = monthTotals[month] + amount
        if cat_ns.Internal not in cats:
            if amount > 0:
                incomeByMonth[month] = incomeByMonth[month] + amount
                income = income + amount
            else:
                outgoingsByMonth[month] = outgoingsByMonth[month] + amount
                outgoings = outgoings + amount

        normalCats = []  # For this item
        for c in cats:
            totals[c] = totals.get(c, 0) + amount
            byMonth[c] = byMonth.get(c, [0] * numberOfMonths)
            count[c] = count.get(c, 0) + 1
            byMonth[c][month] = byMonth[c][month] + amount
            if c not in specialCategories:
                normalCats.append(c)
        bottomCats = normalCats[:] # Copy
        for b in normalCats:
            sups = kb.each(subj=b, pred=rdfs.subClassOf)
            for sup in sups:
                if sup in bottomCats:
                    bottomCats.remove(sup)
        if len(bottomCats) == 0:
           noteError("No categoriy: %s  for <%s>"  # all cats: %s, raw cats:%s"
                        %(`cats`, `s`))  #  ,`cats`, `kb.each(subj=s, pred=rdf.type)`)
        elif bottomCats[0] not in bottomCategories and (bottomCats[0] not in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]):
           noteError("Be more specifc: %s for <%s>"  %(`bottomCats[0]`, `s`)) # Won't get shown e.g. in year-cat.html
        if len(bottomCats) > 1:
           noteError("Inconsistent categories: %s"  # all cats: %s, raw cats:%s"
                        %(`bottomCats`))  #  ,`cats`, `kb.each(subj=s, pred=rdf.type)`)

    
    print '<html xmlns="http://www.w3.org/1999/xhtml">'

    if '--summary' in sys.argv:
        title = "Monthly summary"
    elif '--issues' in sys.argv:
        title = "Issues"
    else:
        title = "Report"

    print """<head>
        <meta charset='UTF-8'>
        <title>%s</title>
        <link rel="Stylesheet" href="report.css">
    </head>
    <body>
    """ % (title)
#           <img src="sand-dollar.gif" alt="dollar" align="right"/>
    


    version = "$Id: fin.py,v 1.31 2014-07-23 13:23:05 timbl Exp $"


#  SUMMARY  TABLE OF CATEGORY BY MONTH

    if '--summary' in sys.argv:
	print "<h2>Personal categories and months %s - %s</h2>" % (startDate, endDate)
	print "<table class='wide' style='border-collapse:collapse; border: 0.01em solid #aaa; text-align: right' ><col style='text-align: left'>"
	
	print "<tr><th></th><th>Total </th>" 
	for month in range(numberOfMonths):
	    m = month + int(startDate[5:7]) - 1
	    while m > 11: m -= 12  # Modulo in python?
	    
	    
	    print "<th><a href='year-chron.html#m%s'>%s</a></th>" %(("0"+`m+1`)[-2:], monthName[m]),
	print "</tr>"
	
	
	def listFor(c, depth=0):   # Any, because there could be 2 copies of same list :-(
	    subs = kb.any(subj = c, pred = owl.disjointUnionOf);
	    res = [ (c, depth) ];
	    if subs == None:
		subs = kb.each(pred = rdfs.subClassOf, obj = c);
		if len(subs) > 0:
		    sys.stderr.write( "Warning: for %s: no disjointUnionOf but subclasses %s\n" %(`c`, `subs`))
		for sub in subs: res += listFor(sub, depth+1)
	    else:
		for sub in subs: res += listFor(sub, depth+1)
	    return res
	    
	printOrder = listFor(qu.Transaction);
	
	for cat, depth in printOrder:
	    label = kb.the(subj=cat, pred=rdfs.label)
	    if label == None:
		label = `cat`
		sys.stderr.write("@@ No label for "+`cat` +"\n")
	    else:
		label = str(label)
	    anchor = cat.fragid
	    if totals.get(cat, None) != None:
		print monthGridRow(anchor, anchor, totals[cat], byMonth.get(cat, [0] * numberOfMonths),
		    numberOfMonths, indent = depth)

	print "<tr><td colspan='14'> ___  </td></tr>"
	print monthGridRow("Income", None,  income, incomeByMonth, numberOfMonths)
	print monthGridRow("Outgoings", None, outgoings, outgoingsByMonth, numberOfMonths)
	print monthGridRow("Balance", None, income + outgoings, monthTotals, numberOfMonths)

	print "</table>"

    
#  Chart of income stacked up against expenses
    if '--charts' in sys.argv:
	print "<p><a href='chart.svg'><p>Chart of day-day income vs expense</p><img src='chart.svg'></a></p>"
	print "<p><a href='all.svg'><p>Chart of all income vs expense</p><img src='all.svg'></a></p>"

	writeChart(filename = "chart.svg",
	    categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
	    totals = totals, income=income, outgoings=outgoings, shortTerm = 1)

	writeChart(filename = "all.svg",
	    categories = bottomCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing],
        totals = totals, income=income, outgoings=outgoings, shortTerm = 0)


    # Output totals
    
    
    if (totalsFilename):
    
	ko = kb.newFormula()    
	for c in quCategories + [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
	    ko.add(subj=c, pred=qu.total, obj=("%7.2f" % totals.get(c,0)))
	ko.add(subj=qu.Transaction, pred=qu.total, obj=("%7.2f" % (income + outgoings)))
	ko.close()
        fo = open(totalsFilename, "w")
        fo.write(ko.n3String())
        fo.close

    if '--issues' in sys.argv:

	#  Generate a list of errors found
	errstr = ""
	for x, list in errors.items():
	    errstr += transactionRow(x)
	    for e in list: errstr += "<tr><td colspan='4'>"+`e`+"</td></tr>\n"  #  @@@ encode error string
	if errstr:
	    print "<h2>Inconsistencies</h2><table>\n" + errstr + "</table>\n"
	
	# List Unclassified Income and Spending
	
	def transactionList(cat):
	    ts = kb.each(pred = rdf.type, obj = cat)
	    if len(ts) == 0: return ""
	    label = kb.any(cat, rdfs.label)
	    st = '<h2>'+label.value()+'</h2>\n'
	    return st + transactionTable(ts)
	
	for cat in [ qu.UnclassifiedIncome, qu.UnclassifiedOutgoing]:
	    print transactionList(cat)

	print reimbursablesCheck();
	
	internalCheck()

    if 0:
	print "<h2>Tax Categories</h2>"
	taxCategories = kb.each(pred=rdf_type, obj=tax.Category)
	printCategoryTotalsOnly(taxCategories + [ qu.Unclassified], totals, count)

	print "<h2>Tax stuff</h2>"
	print "<table>"
	print "<tr><th>-<th>Form line</th><th>amount</th></tr>"
	print "</table>"
        
	# print "<h2>Personal Category total</h2>"
	# printCategoryTotalsOnly(quCategories + [ qu.Unclassified], totals, count)

	print

	print "Note totals for tax and personal breakdowns must match."
	dates = kb.statementsMatching(pred=qu.date)
	print "There should be a total of %i transactions in each." % len(dates)

    

    if 0:
        print "<pre>(consistency check)"
        problems = 0
        for s in dates:
            tra = s.subject()
            types = kb.each(subj=tra, pred=rdf_type)
            for typ in types:
                if typ is qu.Unclassified or typ is qu.Classified:
                    break # ok
            else:
                print "@@@@ problem transcation with no classified or unclassified, with types", types
                printTransactionDetails(tra)
                problems = problems + 1
        print problems, "problems.</pre>"
    
    print "</body></html>"
    return fatalErrors