def loadFiles(testFiles):
graph = []
WD = "file://" + os.getcwd() + "/"
for fn in testFiles:
if verbose: stderr.write("Loading data from %s\n" % fn)
uri = uripath.join(WD, fn)
inStream = urllib.urlopen(uri)
while 1:
line = inStream.readline()
if line == "": break
# if verbose: stderr.write("%s\n" % line)
m = comment.match(line)
if m != None: continue
m = statement.match(line)
if m == None:
stderr.write("Syntax error: " + line + "\n")
if verbose:
[stderr.write('%2x ' % ord(c)) for c in line]
stderr.write('\n')
exit(-1)
triple = m.group(1), m.group(2), m.group(3)
if verbose:
stderr.write("Triple: %s %s %s.\n" %
(triple[0], triple[1], triple[2]))
graph.append(triple)
if verbose:
stderr.write("\nThere are %i statements in graph\n" % (len(graph)))
return graph
def parseFile(self, name):
graph = []
WD = "file://" + os.getcwd() + "/"
if verbose: stderr.write("Loading data from %s\n" % name)
uri = uripath.join(WD, name)
inStream = urllib.urlopen(uri)
for line in inStream:
if line == "": break
# if verbose: stderr.write("%s\n" % line)
m = comment.match(line)
if m != None: continue
m = statement.match(line)
if m == None:
stderr.write("Syntax error: " + line + "\n")
if verbose:
[stderr.write('%2x ' % ord(c)) for c in line]
stderr.write('\n')
exit(-1)
triple = m.group(1), m.group(2), m.group(3)
if verbose:
stderr.write("Triple: %s %s %s.\n" %
(triple[0], triple[1], triple[2]))
yield triple
def loadFiles(testFiles):
graph = []
WD = "file://" + os.getcwd() + "/"
for fn in testFiles:
if verbose: stderr.write("Loading data from %s\n" % fn)
uri = uripath.join(WD, fn)
inStream = urllib.urlopen(uri)
while 1:
line = inStream.readline()
if line == "" : break
# if verbose: stderr.write("%s\n" % line)
m = comment.match(line)
if m != None: continue
m = statement.match(line)
if m == None:
stderr.write("Syntax error: "+line+"\n")
if verbose:
[stderr.write('%2x ' % ord(c)) for c in line]
stderr.write('\n')
exit(-1)
triple = m.group(1), m.group(2), m.group(3)
if verbose: stderr.write( "Triple: %s %s %s.\n" % (triple[0], triple[1], triple[2]))
graph.append(triple)
if verbose: stderr.write("\nThere are %i statements in graph\n" % (len(graph)))
return graph
def parseFile(self, name):
graph = []
WD = "file://" + os.getcwd() + "/"
if verbose:
stderr.write("Loading data from %s\n" % name)
uri = uripath.join(WD, name)
inStream = urllib.urlopen(uri)
for line in inStream:
if line == "":
break
# if verbose: stderr.write("%s\n" % line)
m = comment.match(line)
if m != None:
continue
m = statement.match(line)
if m == None:
stderr.write("Syntax error: " + line + "\n")
if verbose:
[stderr.write("%2x " % ord(c)) for c in line]
stderr.write("\n")
exit(-1)
triple = m.group(1), m.group(2), m.group(3)
if verbose:
stderr.write("Triple: %s %s %s.\n" % (triple[0], triple[1], triple[2]))
yield triple
def testParser(command, kb, output, errorFile):
"""The main parser tester
"""
temp_adder = md5it(command)
commandNode = output.newBlankNode()
output.add(commandNode, rdf.type, n3test.N3Parser)
output.add(commandNode, n3test.command, command)
for test in serial(gatherDAWGStyleTests(kb), gatherCwmStyleTests(kb)):
name, type, description, inputDocument, outputDocument = test
case = name + temp_adder + ".out" # Make up temp filename
tempFile = output.newSymbol(join(base(), ',temp/' + case))
# if description == None: description = case + " (no description)"
output.add(inputDocument, rdf.type, n3test.Input)
output.add(inputDocument, n3test.expected, type)
output.add(inputDocument, n3test.description, description)
#result = 1
thisCommand = ((command + ' > %s 2>>%s') % \
(inputDocument.uriref(), tempFile.uriref()[5:], errorFile))
result = system(thisCommand)
print thisCommand
if result != 0: # Error case:
output.add(commandNode, n3test.failsParsing, inputDocument)
parseResult = output.newBlankNode()
output.add(inputDocument, commandNode, parseResult)
output.add(parseResult, n3test.isFile, rdf.nil)
ef = open(errorFile, "r")
output.add(parseResult, n3test.errorMessage, ef.read().decode('utf-8'))
ef.close()
else:
output.add(commandNode, n3test.parses, inputDocument)
parseResult = output.newBlankNode()
output.add(inputDocument, commandNode, parseResult)
output.add(parseResult, n3test.isFile, tempFile)
if outputDocument is None:
output.add(parseResult, n3test.doesNotMatch, rdf.nil)
else:
a = output.newBlankNode()
child_stdin, child_stdout = popen4("%s %s -f %s -d %s" % \
('python', '$SWAP/cant.py', tempFile.uriref(), outputDocument.uriref()))
output.add(a, rdf.type, n3test.Diff)
try:
ds = "".join([escapize(ii) for ii in child_stdout.read().decode('utf-8')])
except UnicodeDecodeError:
ds = "unicode decode error in diff"
output.add(a, n3test.diffString, ds)
output.add(parseResult, a, outputDocument)
def main(argv):
"""Usage: n3absyn.py foo.n3 --pprint | --lisp | --rif | --mathml
--pprint to print the JSON structure using python's pretty printer
--lisp to print a lisp s-expression for use with ACL2
--rif for Rule Interchange Format (RIF)
--mathml for MathML
"""
path = argv[1]
addr = uripath.join(uripath.base(), path)
kb = llyn.RDFStore()
f = kb.load(addr)
j = json_formula(f)
if '--pprint' in argv:
import pprint
pprint.pprint(j)
elif '--lisp' in argv:
for s in lisp_form(j):
sys.stdout.write(s)
elif '--ikl' in argv:
for s in ikl_sentence(j, []):
sys.stdout.write(s)
elif '--rif' in argv:
for s in xml_form(j):
sys.stdout.write(s)
elif '--mathml' in argv:
for s in mathml_top():
sys.stdout.write(s)
for s in mathml_form(j):
sys.stdout.write(s)
elif '--defns' in argv:
# split top-level N3 constructs
assert (j[0]) == 'forall'
assert (j[2][0]) == 'and'
for s in mathml_top():
sys.stdout.write(s)
sys.stdout.write('<html xmlns="http://www.w3.org/1999/xhtml">\n')
sys.stdout.write('<head><title>defns</title></head>\n')
sys.stdout.write('<body>\n')
sys.stdout.write("<ul>\n")
for expr in j[2][1:]:
sys.stdout.write("<li>\n")
for s in mathml_form(expr):
sys.stdout.write(s)
sys.stdout.write("</li>\n")
sys.stdout.write("</ul>\n")
sys.stdout.write("</body>\n")
sys.stdout.write("</html>\n")
def main(argv):
"""Usage: n3absyn.py foo.n3 --pprint | --lisp | --rif | --mathml
--pprint to print the JSON structure using python's pretty printer
--lisp to print a lisp s-expression for use with ACL2
--rif for Rule Interchange Format (RIF)
--mathml for MathML
"""
path = argv[1]
addr = uripath.join(uripath.base(), path)
kb = llyn.RDFStore()
f = kb.load(addr)
j = json_formula(f)
if '--pprint' in argv:
import pprint
pprint.pprint(j)
elif '--lisp' in argv:
for s in lisp_form(j):
sys.stdout.write(s)
elif '--ikl' in argv:
for s in ikl_sentence(j, []):
sys.stdout.write(s)
elif '--rif' in argv:
for s in xml_form(j):
sys.stdout.write(s)
elif '--mathml' in argv:
for s in mathml_top():
sys.stdout.write(s)
for s in mathml_form(j):
sys.stdout.write(s)
elif '--defns' in argv:
# split top-level N3 constructs
assert(j[0]) == 'forall'
assert(j[2][0]) == 'and'
for s in mathml_top():
sys.stdout.write(s)
sys.stdout.write('<html xmlns="http://www.w3.org/1999/xhtml">\n')
sys.stdout.write('<head><title>defns</title></head>\n')
sys.stdout.write('<body>\n')
sys.stdout.write("<ul>\n")
for expr in j[2][1:]:
sys.stdout.write("<li>\n")
for s in mathml_form(expr):
sys.stdout.write(s)
sys.stdout.write("</li>\n")
sys.stdout.write("</ul>\n")
sys.stdout.write("</body>\n")
sys.stdout.write("</html>\n")
def main(args):
if not args[1:]:
usage()
sys.exit(1)
addr = uripath.join("file:" + os.getcwd() + "/", args[1])
progress("loading...", addr)
kb = load(addr)
progress("exporting...")
extractLDIF(kb)
def main(args):
if not args[1:]:
usage()
sys.exit(1)
addr = uripath.join("file:" + os.getcwd() + "/", args[1])
progress("loading...", addr)
kb = load(addr)
progress("exporting...")
extractLDIF(kb)
def main(args):
if not args[1:]:
usage()
sys.exit(1)
c = CalWr(sys.stdout.write)
if args[3:] and args[1] == '--floattz':
tz = args[2]
c.floatTZ = tz
del args[1:3]
addr = uripath.join("file:" + os.getcwd() + "/", args[1])
progress("loading...", addr)
sts = load(addr)
progress("exporting...")
c.export(sts, addr)
def main(args):
if not args[1:]:
usage()
sys.exit(1)
c = CalWr(sys.stdout.write)
if args[3:] and args[1] == '--floattz':
tz = args[2]
c.floatTZ = tz
del args[1:3]
addr = uripath.join("file:" + os.getcwd() + "/", args[1])
progress("loading...", addr)
sts = load(addr)
progress("exporting...")
c.export(sts, addr)
def main():
global already, agenda, errors
parseAs = None
grammarFile = None
parseFile = None
yaccFile = None
global verbose
global g
verbose = 0
lumped = 1
try:
opts, args = getopt.getopt(sys.argv[1:], "ha:v:p:g:y:",
["help", "as=", "verbose=", "parse=", "grammar=", "yacc="])
except getopt.GetoptError:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose =int(a)
diag.chatty_flag = int(a)
if o in ("-a", "--as"):
parseAs = uripath.join(uripath.base(), a)
if o in ("-p", "--parse"):
parseFile = uripath.join(uripath.base(), a)
if o in ("-g", "--grammar"):
grammarFile = uripath.join(uripath.base(), a)
if o in ("-y", "--yacc"):
yaccFile = uripath.join(uripath.base(), a)[5:] # strip off file:
# if testFiles == []: testFiles = [ "/dev/stdin" ]
if not parseAs:
usage()
sys.exit(2)
parseAs = uripath.join(uripath.base(), parseAs)
if not grammarFile:
grammarFile = parseAs.split("#")[0] # strip off fragid
else:
grammarFile = uripath.join(uripath.base(), grammarFile)
# The Grammar formula
progress("Loading " + grammarFile)
start = clock()
g = load(grammarFile)
taken = clock() - start + 1
progress("Loaded %i statements in %fs, ie %f/s." %
(len(g), taken, len(g)/taken))
document = g.newSymbol(parseAs)
already = []
agenda = []
errors = []
doProduction(document)
while agenda:
x = agenda[0]
agenda = agenda[1:]
already.append(x)
doProduction(x)
if errors != []:
progress("###### FAILED with %i errors." % len(errors))
for s in errors: progress ("\t%s" % s)
exit(-2)
else:
progress( "Ok for predictive parsing")
#if parser.verb: progress "Branch table:", branchTable
if verbose:
progress( "Literal terminals: %s" % literalTerminals.keys())
progress("Token regular expressions:")
for r in tokenRegexps:
progress( "\t%s matches %s" %(r, tokenRegexps[r].pattern) )
if yaccFile:
yacc=open(yaccFile, "w")
yaccConvert(yacc, document, tokenRegexps)
yacc.close()
if parseFile == None: exit(0)
ip = webAccess.urlopenForRDF(parseFile, None)
str = ip.read().decode('utf_8')
sink = g.newFormula()
keywords = g.each(pred=BNF.keywords, subj=document)
keywords = [a.value() for a in keywords]
p = PredictiveParser(sink=sink, top=document, branchTable= branchTable,
tokenRegexps= tokenRegexps, keywords = keywords)
p.verb = verbose
start = clock()
p.parse(str)
taken = clock() - start + 1
progress("Loaded %i chars in %fs, ie %f/s." %
(len(str), taken, len(str)/taken))
progress("Parsed <%s> OK" % parseFile)
sys.exit(0) # didn't crash
def doCommand(serialDevice=None, outputURI=None, doTracks=1, doWaypoints=1, verbose=0):
if os.name == 'nt':
if not serialDevice: serialDevice = "com1"
phys = Win32SerialLink(serialDevice)
else:
if not serialDevice: serialDevice = "/dev/ttyS0"
# serialDevice = "/dev/cu.USA19H191P1.1"
phys = UnixSerialLink(serialDevice)
gps = Garmin(phys)
print "GPS Product ID: %d Descriptions: %s Software version: %2.2f" % \
(gps.prod_id, gps.prod_descs, gps.soft_ver)
f = formula() # Empty store of RDF data
base = uripath.base()
record = f.newBlankNode()
f.add(record, RDF.type, GPS.Record)
if doWaypoints:
# show waypoints
if verbose: print "Getting waypoints"
wpts = gps.getWaypoints()
for w in wpts:
if verbose: progress(`w`)
wpt = symbol(uripath.join(base, w.ident))
f.add(record, GPS.waypoint, wpt)
f.add(wpt, WGS.lat, obj=intern(degrees(w.slat)))
f.add(wpt, WGS.long, obj=intern(degrees(w.slon)))
if doTracks:
# show track
if verbose: print "Getting tracks"
tracks = gps.getTracks()
for t in tracks:
track = f.newBlankNode()
f.add(record, GPS.track, track)
for p in t:
if isinstance(p, TrackHdr):
if verbose: progress(`p`)
f.add(track, GPS.disp, intern(p.dspl))
f.add(track, GPS.color, intern(p.color))
f.add(track, GPS.trk_ident, intern(p.trk_ident))
else:
if verbose: progress(`p`)
point = f.newBlankNode()
f.add(track, GPS.trackpoint, point)
f.add(point, WGS.lat, obj=intern(degrees(p.slat)))
f.add(point, WGS.long, obj=intern(degrees(p.slon)))
# if verbose: progress(" time=", p.time)
# progress('p.time='+`p.time`) # @@
if p.time == 0 or p.time == 0xffffffffL:
if verbose: progress("time=%8x, ignoring" % p.time)
else:
f.add(point, WGS.time, obj=intern(isodate.fullString(TimeEpoch+p.time)))
phys.f.close() # Should really be done by the del() below, but isn't
del(phys) # close serial link (?)
f = f.close()
if verbose:
progress("Beginning output. You can disconnect the GPS now.")
s = f.n3String(base=base, flags="d") # Flag - no default prefix, preserve gps: prefix hint
if outputURI != None:
op = open(outputURI, "w")
op.write(s)
op.close()
else:
print s
def testCwmSparql(kb, output, errorFile):
"""The main parser tester
"""
temp_adder = '_dawg_test'
commandNode = output.newBlankNode()
thisProgram = output.newSymbol('http://www.w3.org/2000/10/swap/test/sparql/dawg_tester.py')
cwmURI = output.newSymbol('http://www.w3.org/2000/10/swap/doc/cwm#')
testCount = 0
for test in gatherDAWGStyleTests(kb):
testCount += 1
## if testCount < 317:
## continue
testURI, name, type, description, queryDocument, inputDocument, outputDocument = test
print('%s %s\t%s\t%s\t%s\t%s' % (testCount, testURI, name, description, queryDocument.uriref(), type))
case = (name + temp_adder + ".out").replace(' ',
'_').replace('\n',
'_').replace('\t',
'_').replace('|',
'_').replace('\\',
'_').replace('/',
'_').replace('&',
'_').replace("'",
'_').replace('"',
'_') # Make up temp filename
tempFile = output.newSymbol(join(base(), ',temp/' + case))
if type == 'Query':
if inputDocument is None:
inputDocument = lambda : 'empty.n3'
inputDocument.uriref = inputDocument
try:
inputDocument.uriref()
except:
raise ValueError(inputDocument)
thisCommand = ('python ../../cwm.py %s --sparql=%s --sparqlResults > %s' %
(inputDocument.uriref(), queryDocument.uriref(),
tempFile.uriref()[5:]))
print thisCommand
result = system(thisCommand)
if result != 0:
result = earl.fail
else:
if outputDocument.uriref()[-3:] == 'srx': # sparql results format. how do we deal with that?
resultString = sparqlResults2Turtle(outputDocument.uriref())
outputDocument = output.newSymbol(tempFile.uriref() + '2')
tempFile2 = outputDocument.uriref()[5:]
temp2 = file(tempFile2, 'w')
try:
temp2.write(resultString)
finally:
temp2.close()
temp = file(tempFile.uriref()[5:], 'r')
try:
tempString = temp.read()
finally:
temp.close()
if 'sparql xmlns="http://www.w3.org/2005/sparql-results#"' in tempString:
resultString = sparqlResults2Turtle(tempFile.uriref())
tempFile = output.newSymbol(tempFile.uriref() + '3')
tempFile2 = tempFile.uriref()[5:]
temp2 = file(tempFile2, 'w')
try:
temp2.write(resultString)
finally:
temp2.close()
else:
resultString = output.store.load(tempFile.uriref(), contentType="application/rdf+xml").ntString()
tempFile = output.newSymbol(tempFile.uriref() + '3')
tempFile2 = tempFile.uriref()[5:]
temp2 = file(tempFile2, 'w')
try:
temp2.write(resultString)
finally:
temp2.close()
result = system('python ../../cwm.py %s --ntriples | python ../../cant.py -d %s' %
(outputDocument.uriref(), tempFile.uriref()))
if result == 0:
result = earl['pass']
else:
result = earl['fail']
else:
thisCommand = ('python ../../cwm.py --language=sparql %s > /dev/null 2> /dev/null' %
(queryDocument.uriref(), ))
result = system(thisCommand)
if (result == 0 and type == 'Positive') or \
(result != 0 and type == 'Negative'):
result = earl['pass']
else:
system('echo %s >> %s' % (thisCommand, errorFile))
thisCommand = ('python ../../cwm.py --language=sparql %s > /dev/null 2>> %s' %
(queryDocument.uriref(), errorFile))
system(thisCommand)
result = earl['fail']
caseURI = output.newBlankNode()
output.add(caseURI, rdf.type, earl.Assertion)
output.add(caseURI, earl.assertedBy, thisProgram)
output.add(caseURI, earl.subject, cwmURI)
output.add(caseURI, earl.test, testURI)
resultURI = output.newBlankNode()
output.add(caseURI, earl.result, resultURI)
output.add(resultURI, rdf.type, earl.TestResult)
output.add(resultURI, earl.outcome, result)
if outputDocument:
output.add(resultURI, mf.result, outputDocument)
output.add(resultURI, mf.got, tempFile)
print '\t\t\tresult\t', result
def doCommand():
"""Command line RDF/N3 tool
<command> <options> <steps> [--with <more args> ]
options:
--pipe Don't store, just pipe out *
steps, in order left to right:
--rdf Input & Output ** in RDF/XML insead of n3 from now on
--n3 Input & Output in N3 from now on. (Default)
--rdf=flags Input & Output ** in RDF and set given RDF flags
--n3=flags Input & Output in N3 and set N3 flags
--ntriples Input & Output in NTriples (equiv --n3=usbpartane -bySubject -quiet)
--language=x Input & Output in "x" (rdf, n3, etc) --rdf same as: --language=rdf
--languageOptions=y --n3=sp same as: --language=n3 --languageOptions=sp
--ugly Store input and regurgitate, data only, fastest *
--bySubject Store input and regurgitate in subject order *
--no No output *
(default is to store and pretty print with anonymous nodes) *
--base=<uri> Set the base URI. Input or output is done as though theis were the document URI.
--closure=flags Control automatic lookup of identifiers (see below)
<uri> Load document. URI may be relative to current directory.
--apply=foo Read rules from foo, apply to store, adding conclusions to store
--patch=foo Read patches from foo, applying insertions and deletions to store
--filter=foo Read rules from foo, apply to store, REPLACING store with conclusions
--query=foo Read a N3QL query from foo, apply it to the store, and replace the store with its conclusions
--sparql=foo Read a SPARQL query from foo, apply it to the store, and replace the store with its conclusions
--rules Apply rules in store to store, adding conclusions to store
--think as -rules but continue until no more rule matches (or forever!)
--engine=otter use otter (in your $PATH) instead of llyn for linking, etc
--why Replace the store with an explanation of its contents
--why=u proof tries to be shorter
--mode=flags Set modus operandi for inference (see below)
--reify Replace the statements in the store with statements describing them.
--dereify Undo the effects of --reify
--flatten Reify only nested subexpressions (not top level) so that no {} remain.
--unflatten Undo the effects of --flatten
--think=foo as -apply=foo but continue until no more rule matches (or forever!)
--purge Remove from store any triple involving anything in class log:Chaff
--data Remove all except plain RDF triples (formulae, forAll, etc)
--strings Dump :s to stdout ordered by :k whereever { :k log:outputString :s }
--crypto Enable processing of crypto builtin functions. Requires python crypto.
--help print this message
--revision print CVS revision numbers of major modules
--chatty=50 Verbose debugging output of questionable use, range 0-99
--sparqlServer instead of outputting, start a SPARQL server on port 8000 of the store
--sparqlResults After sparql query, print in sparqlResults format instead of rdf
finally:
--with Pass any further arguments to the N3 store as os:argv values
* mutually exclusive
** doesn't work for complex cases :-/
Examples:
cwm --rdf foo.rdf --n3 --pipe Convert from rdf/xml to rdf/n3
cwm foo.n3 bar.n3 --think Combine data and find all deductions
cwm foo.n3 --flat --n3=spart
Mode flags affect inference extedning to the web:
r Needed to enable any remote stuff.
a When reading schema, also load rules pointed to by schema (requires r, s)
E Errors loading schemas of definitive documents are ignored
m Schemas and definitive documents laoded are merged into the meta knowledge
(otherwise they are consulted independently)
s Read the schema for any predicate in a query.
u Generate unique ids using a run-specific
Closure flags are set to cause the working formula to be automatically exapnded to
the closure under the operation of looking up:
s the subject of a statement added
p the predicate of a statement added
o the object of a statement added
t the object of an rdf:type statement added
i any owl:imports documents
r any doc:rules documents
E errors are ignored --- This is independant of --mode=E
n Normalize IRIs to URIs
e Smush together any nodes which are = (owl:sameAs)
See http://www.w3.org/2000/10/swap/doc/cwm for more documentation.
Setting the environment variable CWM_RDFLIB to 1 maked Cwm use rdflib to parse
rdf/xml files. Note that this requires rdflib.
"""
import time
import sys
from swap import myStore
# These would just be attributes if this were an object
global _store
global workingContext
option_need_rdf_sometime = 0 # If we don't need it, don't import it
# (to save errors where parsers don't exist)
option_pipe = 0 # Don't store, just pipe though
option_inputs = []
option_reify = 0 # Flag: reify on output (process?)
option_flat = 0 # Flag: reify on output (process?)
option_crypto = 0 # Flag: make cryptographic algorithms available
setTracking(0)
option_outURI = None
option_outputStyle = "-best"
_gotInput = 0 # Do we not need to take input from stdin?
option_meta = 0
option_normalize_iri = 0
option_flags = { "rdf":"l", "n3":"", "think":"", "sparql":""}
# RDF/XML serializer can't do list ("collection") syntax.
option_quiet = 0
option_with = None # Command line arguments made available to N3 processing
option_engine = "llyn"
option_why = ""
_step = 0 # Step number used for metadata
_genid = 0
hostname = "localhost" # @@@@@@@@@@@ Get real one
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
option_format = "n3" # set the default format
option_first_format = None
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# First pass on command line - - - - - - - P A S S 1
for argnum in range(1,len(sys.argv)): # options after script name
arg = sys.argv[argnum]
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
# _equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
try:
[_lhs,_rhs]=arg.split('=',1)
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
except ValueError: pass
if arg == "-ugly": option_outputStyle = arg
elif _lhs == "-base": option_baseURI = _uri
elif arg == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_need_rdf_sometime = 1
elif _lhs == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_flags["rdf"] = _rhs
option_need_rdf_sometime = 1
elif arg == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
option_flags["n3"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
if "n" in _rhs:
option_normalize_iri = 1
#elif _lhs == "-solve":
# sys.argv[argnum+1:argnum+1] = ['-think', '-filter=' + _rhs]
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _rhs
elif arg == "-quiet": option_quiet = 1
elif arg == "-pipe": option_pipe = 1
elif arg == "-crypto": option_crypto = 1
elif _lhs == "-why":
diag.tracking=1
diag.setTracking(1)
option_why = _rhs
elif arg == "-why":
diag.tracking=1
diag.setTracking(1)
option_why = ""
elif arg == "-track":
diag.tracking=1
diag.setTracking(1)
elif arg == "-bySubject": option_outputStyle = arg
elif arg == "-no": option_outputStyle = "-no"
elif arg == "-debugString": option_outputStyle = "-debugString"
elif arg == "-strings": option_outputStyle = "-no"
elif arg == "-sparqlResults": option_outputStyle = "-no"
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "bravestpun"
option_outputStyle = "-bySubject"
option_quiet = 1
elif _lhs == "-outURI": option_outURI = _uri
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-apply=": pass
elif arg[:7] == "-patch=": pass
elif arg == "-reify": option_reify = 1
elif arg == "-flat": option_flat = 1
elif arg == "-help":
print doCommand.__doc__
print notation3.ToN3.flagDocumentation
print toXML.ToRDF.flagDocumentation
try:
from swap import sax2rdf # RDF1.0 syntax parser to N3 RDF stream
print sax2rdf.RDFXMLParser.flagDocumentation
except:
pass
return
elif arg == "-revision":
progress( "cwm=",cvsRevision, "llyn=", llyn.cvsRevision)
return
elif arg == "-with":
option_with = sys.argv[argnum+1:] # The rest of the args are passed to n3
break
elif arg[0] == "-": pass # Other option
else :
option_inputs.append(join(option_baseURI, arg))
_gotInput = _gotInput + 1 # input filename
# Between passes, prepare for processing
setVerbosity(0)
if not option_normalize_iri:
llyn.canonical = lambda x: x
# Base defauts
if option_baseURI == _baseURI: # Base not specified explicitly - special case
if _outURI == _baseURI: # Output name not specified either
if _gotInput == 1: # But input file *is*,
_outURI = option_inputs[0] # Just output to same URI
option_baseURI = _outURI # using that as base.
if diag.tracking:
_outURI = RDFSink.runNamespace()[:-1]
option_baseURI = _outURI
option_baseURI = splitFrag(option_baseURI)[0]
# Fix the output sink
if option_format == "rdf":
_outSink = toXML.ToRDF(sys.stdout, _outURI, base=option_baseURI, flags=option_flags["rdf"])
elif option_format == "n3" or option_format == "sparql":
_outSink = notation3.ToN3(sys.stdout.write, base=option_baseURI,
quiet=option_quiet, flags=option_flags["n3"])
elif option_format == "trace":
_outSink = RDFSink.TracingRDFSink(_outURI, base=option_baseURI,
flags=option_flags.get("trace",""))
if option_pipe:
# this is really what a parser wants to dump to
_outSink.backing = llyn.RDFStore( _outURI+"#_g",
argv=option_with, crypto=option_crypto)
else:
# this is really what a store wants to dump to
_outSink.backing = notation3.ToN3(sys.stdout.write,
base=option_baseURI, quiet=option_quiet,
flags=option_flags["n3"])
# hm. why does TimBL use sys.stdout.write, above? performance at the
else:
raise NotImplementedError
version = "$Id: cwm.py,v 1.197 2007/12/13 15:38:39 syosi Exp $"
if not option_quiet and option_outputStyle != "-no":
_outSink.makeComment("Processed by " + version[1:-1]) # Strip $ to disarm
_outSink.makeComment(" using base " + option_baseURI)
if option_flat:
_outSink = notation3.Reifier(_outSink, _outURI+ "#_formula", flat=1)
if diag.tracking:
myReason = BecauseOfCommandLine(`sys.argv`)
# @@ add user, host, pid, pwd, date time? Privacy!
else:
myReason = None
if option_pipe:
_store = _outSink
workingContext = _outSink #.newFormula()
else:
if "u" in option_flags["think"]:
_store = llyn.RDFStore(argv=option_with, crypto=option_crypto)
else:
_store = llyn.RDFStore( _outURI+"#_g",
argv=option_with, crypto=option_crypto)
myStore.setStore(_store)
if _gotInput:
workingContext = _store.newFormula(option_inputs [0]+"#_work")
newTopLevelFormula(workingContext)
else: # default input
if option_first_format is None: option_first_format = option_format
ContentType={ "rdf": "application/xml+rdf", "n3":
"text/rdf+n3", "sparql":
"x-application/sparql"}[option_first_format]
workingContext = _store.load(
# asIfFrom = join(_baseURI, ".stdin"),
asIfFrom = _baseURI,
contentType = ContentType,
flags = option_flags[option_first_format],
remember = 0,
referer = "",
why = myReason, topLevel=True)
workingContext.reopen()
workingContext.stayOpen = 1 # Never canonicalize this. Never share it.
# ____________________________________________________________________
# Take commands from command line:- - - - - P A S S 2
option_format = "n3" # Use RDF/n3 rather than RDF/XML
option_flags = { "rdf":"l", "n3":"", "think": "", "sparql":"" }
option_quiet = 0
_outURI = _baseURI
option_baseURI = _baseURI # To start with
def filterize():
"""implementation of --filter
for the --filter command, so we don't have it printed twice
"""
global workingContext
global r
workingContext = workingContext.canonicalize()
_store._formulaeOfLength = {}
filterContext = _store.newFormula()
newTopLevelFormula(filterContext)
_store.load(_uri, openFormula=filterContext,
why=myReason, referer="")
_newContext = _store.newFormula()
newTopLevelFormula(_newContext)
applyRules(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
sparql_query_formula = None
for arg in sys.argv[1:]: # Command line options after script name
if verbosity()>5: progress("Processing %s." % (arg))
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
_equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
if _equals >=0:
_lhs = arg[:_equals]
_rhs = arg[_equals+1:]
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri =_rhs
if arg[0] != "-":
_inputURI = join(option_baseURI, splitFrag(arg)[0])
assert ':' in _inputURI
ContentType={ "rdf": "application/xml+rdf", "n3":
"text/rdf+n3",
"sparql": "x-application/sparql"}[option_format]
if not option_pipe: workingContext.reopen()
try:
load(_store, _inputURI,
openFormula=workingContext,
contentType =ContentType,
flags=option_flags[option_format],
referer="",
why=myReason)
except:
progress(_inputURI)
raise
_gotInput = 1
elif arg == "-help":
pass # shouldn't happen
elif arg == "-revision":
pass
elif _lhs == "-base":
option_baseURI = _uri
if verbosity() > 10: progress("Base now "+option_baseURI)
elif arg == "-ugly":
option_outputStyle = arg
elif arg == "-crypto": pass
elif arg == "-pipe": pass
elif _lhs == "-outURI": option_outURI = _uri
elif arg == "-rdf": option_format = "rdf"
elif _lhs == "-rdf":
option_format = "rdf"
option_flags["rdf"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
workingContext.setClosureMode(_rhs)
elif arg == "-n3": option_format = "n3"
elif _lhs == "-n3":
option_format = "n3"
option_flags["n3"] = _rhs
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _lhs
elif arg == "-quiet" : option_quiet = 1
elif _lhs == "-chatty": setVerbosity(int(_rhs))
elif arg[:7] == "-track=":
diag.tracking = int(_rhs)
elif option_pipe: ############## End of pipable options
print "# Command line error: %s illegal option with -pipe", arg
break
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "spartan"
option_outputStyle = "-bySubject"
option_quiet = 1
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-debugString":
option_outputStyle = arg
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext);
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext);
elif arg[:7] == "-patch=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
workingContext.reopen()
patch(workingContext, filterContext);
elif _lhs == "-filter":
filterize()
elif _lhs == "-query":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
_newContext = _store.newFormula()
applyQueries(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
elif _lhs == "-sparql":
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri, why=myReason,
referer="", contentType="x-application/sparql")
_newContext = _store.newFormula()
_newContext.stayOpen = True
sparql_query_formula = filterContext
applySparqlQueries(workingContext, filterContext, _newContext)
# workingContext.close()
workingContext = _newContext
elif _lhs == "-why" or arg == "-why":
workingContext.stayOpen = False
workingContext = workingContext.close()
workingContext = explainFormula(workingContext, option_why)
# Can't prove proofs
diag.tracking=0
diag.setTracking(0)
elif arg == "-dump":
workingContext = workingContext.canonicalize()
progress("\nDump of working formula:\n" + workingContext.debugString())
elif arg == "-purge":
workingContext.reopen()
_store.purge(workingContext)
elif arg == "-purge-rules" or arg == "-data":
workingContext.reopen()
_store.purgeExceptData(workingContext)
elif arg == "-rules":
workingContext.reopen()
applyRules(workingContext, workingContext)
elif arg[:7] == "-think=":
filterContext = _store.load(_uri, referer="", why=myReason, topLevel=True)
if verbosity() > 4:
progress( "Input rules to --think from " + _uri)
workingContext.reopen()
think(workingContext, filterContext, mode=option_flags["think"])
elif arg[:7] == "-solve=":
# --solve is a combination of --think and --filter.
think(workingContext, mode=option_flags["think"])
filterize()
elif _lhs == "-engine":
option_engine = _rhs
elif arg == "-think":
workingContext.isWorkingContext = True
think(workingContext, mode=option_flags["think"])
elif arg == '-rete':
from swap import pycwmko
pythink = pycwmko.directPychinkoQuery(workingContext)
#return
#pythink()
"""
from pychinko import interpreter
from swap.set_importer import Set, ImmutableSet
pyf = pycwmko.N3Loader.N3Loader()
conv = pycwmko.ToPyStore(pyf)
conv.statements(workingContext)
interp = interpreter.Interpreter(pyf.rules[:])
interp.addFacts(Set(pyf.facts), initialSet=True)
interp.run()
pyf.facts = interp.totalFacts
workingContext = workingContext.store.newFormula()
reconv = pycwmko.FromPyStore(workingContext, pyf)
reconv.run()
"""
elif arg == '-sparqlServer':
from swap.sparql import webserver
from swap import cwm_sparql
sandBoxed(True)
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
def _handler(s):
return cwm_sparql.sparql_queryString(workingContext, s)
webserver.sparql_handler = _handler
webserver.run()
elif arg == "-lxkbdump": # just for debugging
raise NotImplementedError
elif arg == "-lxfdump": # just for debugging
raise NotImplementedError
elif _lhs == "-prove":
# code copied from -filter without really being understood -sdh
_tmpstore = llyn.RDFStore( _outURI+"#_g", metaURI=_metaURI, argv=option_with, crypto=option_crypto)
tmpContext = _tmpstore.newFormula(_uri+ "#_formula")
_newURI = join(_baseURI, "_w_"+`_genid`) # Intermediate
_genid = _genid + 1
_newContext = _tmpstore.newFormula(_newURI+ "#_formula")
_tmpstore.loadURI(_uri)
print targetkb
elif arg == "-flatten":
#raise NotImplementedError
from swap import reify
workingContext = reify.flatten(workingContext)
elif arg == "-unflatten":
from swap import reify
workingContext = reify.unflatten(workingContext)
#raise NotImplementedError
elif arg == "-reify":
from swap import reify
workingContext = reify.reify(workingContext)
elif arg == "-dereify":
from swap import reify
workingContext = reify.dereify(workingContext)
elif arg == "-size":
progress("Size: %i statements in store, %i in working formula."
%(_store.size, workingContext.size()))
elif arg == "-strings": # suppress output
workingContext.outputStrings()
option_outputStyle = "-no"
elif arg == '-sparqlResults':
from cwm_sparql import outputString, SPARQL_NS
ns = _store.newSymbol(SPARQL_NS)
if not sparql_query_formula:
raise ValueError('No query')
else:
sys.stdout.write(outputString(sparql_query_formula, workingContext)[0].encode('utf_8'))
option_outputStyle = "-no"
elif arg == "-no": # suppress output
option_outputStyle = arg
elif arg[:8] == "-outURI=": pass
elif arg == "-with": break
else:
progress( "cwm: Unknown option: " + arg)
sys.exit(-1)
# Squirt it out if not piped
workingContext.stayOpen = 0 # End its use as an always-open knoweldge base
if option_pipe:
workingContext.endDoc()
else:
if hasattr(_outSink, "serializeKB"):
raise NotImplementedError
else:
if verbosity()>5: progress("Begining output.")
workingContext = workingContext.close()
assert workingContext.canonical != None
if option_outputStyle == "-ugly":
_store.dumpChronological(workingContext, _outSink)
elif option_outputStyle == "-bySubject":
_store.dumpBySubject(workingContext, _outSink)
elif option_outputStyle == "-no":
pass
elif option_outputStyle == "-debugString":
print workingContext.debugString()
else: # "-best"
_store.dumpNested(workingContext, _outSink,
flags=option_flags[option_format])
def main():
global already, agenda, errors
parseAs = None
grammarFile = None
parseFile = None
yaccFile = None
global verbose
global g
verbose = 0
lumped = 1
try:
opts, args = getopt.getopt(
sys.argv[1:], "ha:v:p:g:y:",
["help", "as=", "verbose=", "parse=", "grammar=", "yacc="])
except getopt.GetoptError:
usage()
sys.exit(2)
output = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-v", "--verbose"):
verbose = int(a)
diag.chatty_flag = int(a)
if o in ("-a", "--as"):
parseAs = uripath.join(uripath.base(), a)
if o in ("-p", "--parse"):
parseFile = uripath.join(uripath.base(), a)
if o in ("-g", "--grammar"):
grammarFile = uripath.join(uripath.base(), a)
if o in ("-y", "--yacc"):
yaccFile = uripath.join(uripath.base(), a)[5:] # strip off file:
# if testFiles == []: testFiles = [ "/dev/stdin" ]
if not parseAs:
usage()
sys.exit(2)
parseAs = uripath.join(uripath.base(), parseAs)
if not grammarFile:
grammarFile = parseAs.split("#")[0] # strip off fragid
else:
grammarFile = uripath.join(uripath.base(), grammarFile)
# The Grammar formula
progress("Loading " + grammarFile)
start = clock()
g = load(grammarFile)
taken = clock() - start + 1
progress("Loaded %i statements in %fs, ie %f/s." %
(len(g), taken, len(g) / taken))
document = g.newSymbol(parseAs)
already = []
agenda = []
errors = []
doProduction(document)
while agenda:
x = agenda[0]
agenda = agenda[1:]
already.append(x)
doProduction(x)
if errors != []:
progress("###### FAILED with %i errors." % len(errors))
for s in errors:
progress("\t%s" % s)
exit(-2)
else:
progress("Ok for predictive parsing")
#if parser.verb: progress "Branch table:", branchTable
if verbose:
progress("Literal terminals: %s" % literalTerminals.keys())
progress("Token regular expressions:")
for r in tokenRegexps:
progress("\t%s matches %s" % (r, tokenRegexps[r].pattern))
if yaccFile:
yacc = open(yaccFile, "w")
yaccConvert(yacc, document, tokenRegexps)
yacc.close()
if parseFile == None: exit(0)
ip = webAccess.urlopenForRDF(parseFile, None)
lexer = sparql_tokens.Lexer()
lexer.input(ip)
#str = ip.read().decode('utf_8')
sink = g.newFormula()
keywords = g.each(pred=BNF.keywords, subj=document)
keywords = [a.value() for a in keywords]
p = PredictiveParser(sink=sink,
top=document,
branchTable=branchTable,
tokenSet=tokenSet,
keywords=keywords)
p.verb = 1
start = clock()
#print lexer.token()
print p.parse(lexer.token)
taken = clock() - start + 1
# progress("Loaded %i chars in %fs, ie %f/s." %
# (len(str), taken, len(str)/taken))
progress("Parsed <%s> OK" % parseFile)
sys.exit(0) # didn't crash
'file:///devel/dig/': 'http://dig.csail.mit.edu/'
}
service = 'http://pingthesemanticweb.com/rest/?url='
import os, sys # http://pydoc.org/1.6/os.html
from urllib2 import urlopen
import swap
from swap import uripath # http://www.w3.org/2000/10/swap/uripath.py
from swap.uripath import join, URI_unreserved
cwd = os.getcwd()
base = 'file://' + cwd + "/"
for arg in sys.argv[1:]:
uri = join(base, arg)
for local, web in map.items():
uri = uri.replace(local, web)
if uri.endswith('.rdf'): uri = uri[:-4]
if uri.endswith('.n3'): uri = uri[:-3]
print "Ping: ", uri
encoded = ""
for ch in uri:
if ch in URI_unreserved: encoded += ch
else: encoded += "%%%2x" % ord(ch)
# print " with ", service+encoded
x = urlopen(service + encoded)
buf = x.read()
if buf.find('Thanks for') < 0: print buf
x.close()
def doCommand():
"""Command line RDF/N3 tool
<command> <options> <steps> [--with <more args> ]
options:
--pipe Don't store, just pipe out *
steps, in order left to right:
--rdf Input & Output ** in RDF/XML insead of n3 from now on
--n3 Input & Output in N3 from now on. (Default)
--rdf=flags Input & Output ** in RDF and set given RDF flags
--n3=flags Input & Output in N3 and set N3 flags
--ntriples Input & Output in NTriples (equiv --n3=usbpartane -bySubject -quiet)
--language=x Input & Output in "x" (rdf, n3, etc) --rdf same as: --language=rdf
--languageOptions=y --n3=sp same as: --language=n3 --languageOptions=sp
--ugly Store input and regurgitate, data only, fastest *
--bySubject Store input and regurgitate in subject order *
--no No output *
(default is to store and pretty print with anonymous nodes) *
--base=<uri> Set the base URI. Input or output is done as though theis were the document URI.
--closure=flags Control automatic lookup of identifiers (see below)
<uri> Load document. URI may be relative to current directory.
--apply=foo Read rules from foo, apply to store, adding conclusions to store
--patch=foo Read patches from foo, applying insertions and deletions to store
--filter=foo Read rules from foo, apply to store, REPLACING store with conclusions
--query=foo Read a N3QL query from foo, apply it to the store, and replace the store with its conclusions
--sparql=foo Read a SPARQL query from foo, apply it to the store, and replace the store with its conclusions
--rules Apply rules in store to store, adding conclusions to store
--think as -rules but continue until no more rule matches (or forever!)
--engine=otter use otter (in your $PATH) instead of llyn for linking, etc
--why Replace the store with an explanation of its contents
--why=u proof tries to be shorter
--mode=flags Set modus operandi for inference (see below)
--reify Replace the statements in the store with statements describing them.
--dereify Undo the effects of --reify
--flatten Reify only nested subexpressions (not top level) so that no {} remain.
--unflatten Undo the effects of --flatten
--think=foo as -apply=foo but continue until no more rule matches (or forever!)
--purge Remove from store any triple involving anything in class log:Chaff
--data Remove all except plain RDF triples (formulae, forAll, etc)
--strings Dump :s to stdout ordered by :k whereever { :k log:outputString :s }
--crypto Enable processing of crypto builtin functions. Requires python crypto.
--help print this message
--revision print CVS revision numbers of major modules
--chatty=50 Verbose debugging output of questionable use, range 0-99
--sparqlServer instead of outputting, start a SPARQL server on port 8000 of the store
--sparqlResults After sparql query, print in sparqlResults format instead of rdf
finally:
--with Pass any further arguments to the N3 store as os:argv values
* mutually exclusive
** doesn't work for complex cases :-/
Examples:
cwm --rdf foo.rdf --n3 --pipe Convert from rdf/xml to rdf/n3
cwm foo.n3 bar.n3 --think Combine data and find all deductions
cwm foo.n3 --flat --n3=spart
Mode flags affect inference extedning to the web:
r Needed to enable any remote stuff.
a When reading schema, also load rules pointed to by schema (requires r, s)
E Errors loading schemas of definitive documents are ignored
m Schemas and definitive documents laoded are merged into the meta knowledge
(otherwise they are consulted independently)
s Read the schema for any predicate in a query.
u Generate unique ids using a run-specific
Closure flags are set to cause the working formula to be automatically exapnded to
the closure under the operation of looking up:
s the subject of a statement added
p the predicate of a statement added
o the object of a statement added
t the object of an rdf:type statement added
i any owl:imports documents
r any doc:rules documents
E errors are ignored --- This is independant of --mode=E
n Normalize IRIs to URIs
e Smush together any nodes which are = (owl:sameAs)
See http://www.w3.org/2000/10/swap/doc/cwm for more documentation.
Setting the environment variable CWM_RDFLIB to 1 maked Cwm use rdflib to parse
rdf/xml files. Note that this requires rdflib.
"""
import time
import sys
from swap import myStore
# These would just be attributes if this were an object
global _store
global workingContext
option_need_rdf_sometime = 0 # If we don't need it, don't import it
# (to save errors where parsers don't exist)
option_pipe = 0 # Don't store, just pipe though
option_inputs = []
option_reify = 0 # Flag: reify on output (process?)
option_flat = 0 # Flag: reify on output (process?)
option_crypto = 0 # Flag: make cryptographic algorithms available
setTracking(0)
option_outURI = None
option_outputStyle = "-best"
_gotInput = 0 # Do we not need to take input from stdin?
option_meta = 0
option_normalize_iri = 0
option_flags = {"rdf": "l", "n3": "", "think": "", "sparql": ""}
# RDF/XML serializer can't do list ("collection") syntax.
option_quiet = 0
option_with = None # Command line arguments made available to N3 processing
option_engine = "llyn"
option_why = ""
_step = 0 # Step number used for metadata
_genid = 0
hostname = "localhost" # @@@@@@@@@@@ Get real one
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
option_format = "n3" # set the default format
option_first_format = None
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# First pass on command line - - - - - - - P A S S 1
for argnum in range(1, len(sys.argv)): # options after script name
arg = sys.argv[argnum]
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
# _equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
try:
[_lhs, _rhs] = arg.split('=', 1)
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
except ValueError:
pass
if arg == "-ugly": option_outputStyle = arg
elif _lhs == "-base": option_baseURI = _uri
elif arg == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_need_rdf_sometime = 1
elif _lhs == "-rdf":
option_format = "rdf"
if option_first_format == None:
option_first_format = option_format
option_flags["rdf"] = _rhs
option_need_rdf_sometime = 1
elif arg == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-n3":
option_format = "n3"
if option_first_format == None:
option_first_format = option_format
option_flags["n3"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
if "n" in _rhs:
option_normalize_iri = 1
#elif _lhs == "-solve":
# sys.argv[argnum+1:argnum+1] = ['-think', '-filter=' + _rhs]
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _rhs
elif arg == "-quiet":
option_quiet = 1
elif arg == "-pipe":
option_pipe = 1
elif arg == "-crypto":
option_crypto = 1
elif _lhs == "-why":
diag.tracking = 1
diag.setTracking(1)
option_why = _rhs
elif arg == "-why":
diag.tracking = 1
diag.setTracking(1)
option_why = ""
elif arg == "-track":
diag.tracking = 1
diag.setTracking(1)
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-no":
option_outputStyle = "-no"
elif arg == "-debugString":
option_outputStyle = "-debugString"
elif arg == "-strings":
option_outputStyle = "-no"
elif arg == "-sparqlResults":
option_outputStyle = "-no"
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "bravestpun"
option_outputStyle = "-bySubject"
option_quiet = 1
elif _lhs == "-outURI":
option_outURI = _uri
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-apply=":
pass
elif arg[:7] == "-patch=":
pass
elif arg == "-reify":
option_reify = 1
elif arg == "-flat":
option_flat = 1
elif arg == "-help":
print doCommand.__doc__
print notation3.ToN3.flagDocumentation
print toXML.ToRDF.flagDocumentation
try:
from swap import sax2rdf # RDF1.0 syntax parser to N3 RDF stream
print sax2rdf.RDFXMLParser.flagDocumentation
except:
pass
return
elif arg == "-revision":
progress("cwm=", cvsRevision, "llyn=", llyn.cvsRevision)
return
elif arg == "-with":
option_with = sys.argv[argnum +
1:] # The rest of the args are passed to n3
break
elif arg[0] == "-":
pass # Other option
else:
option_inputs.append(join(option_baseURI, arg))
_gotInput = _gotInput + 1 # input filename
# Between passes, prepare for processing
setVerbosity(0)
if not option_normalize_iri:
llyn.canonical = lambda x: x
# Base defauts
if option_baseURI == _baseURI: # Base not specified explicitly - special case
if _outURI == _baseURI: # Output name not specified either
if _gotInput == 1: # But input file *is*,
_outURI = option_inputs[0] # Just output to same URI
option_baseURI = _outURI # using that as base.
if diag.tracking:
_outURI = RDFSink.runNamespace()[:-1]
option_baseURI = _outURI
option_baseURI = splitFrag(option_baseURI)[0]
# Fix the output sink
if option_format == "rdf":
_outSink = toXML.ToRDF(sys.stdout,
_outURI,
base=option_baseURI,
flags=option_flags["rdf"])
elif option_format == "n3" or option_format == "sparql":
_outSink = notation3.ToN3(sys.stdout.write,
base=option_baseURI,
quiet=option_quiet,
flags=option_flags["n3"])
elif option_format == "trace":
_outSink = RDFSink.TracingRDFSink(_outURI,
base=option_baseURI,
flags=option_flags.get("trace", ""))
if option_pipe:
# this is really what a parser wants to dump to
_outSink.backing = llyn.RDFStore(_outURI + "#_g",
argv=option_with,
crypto=option_crypto)
else:
# this is really what a store wants to dump to
_outSink.backing = notation3.ToN3(sys.stdout.write,
base=option_baseURI,
quiet=option_quiet,
flags=option_flags["n3"])
# hm. why does TimBL use sys.stdout.write, above? performance at the
else:
raise NotImplementedError
version = "$Id: cwm.py,v 1.198 2012-01-30 09:30:20 timbl Exp $"
if not option_quiet and option_outputStyle != "-no":
_outSink.makeComment("Processed by " +
version[1:-1]) # Strip $ to disarm
_outSink.makeComment(" using base " + option_baseURI)
if option_flat:
_outSink = notation3.Reifier(_outSink, _outURI + "#_formula", flat=1)
if diag.tracking:
myReason = BecauseOfCommandLine( ` sys.argv `)
# @@ add user, host, pid, pwd, date time? Privacy!
else:
myReason = None
if option_pipe:
_store = _outSink
workingContext = _outSink #.newFormula()
else:
if "u" in option_flags["think"]:
_store = llyn.RDFStore(argv=option_with, crypto=option_crypto)
else:
_store = llyn.RDFStore(_outURI + "#_g",
argv=option_with,
crypto=option_crypto)
myStore.setStore(_store)
if _gotInput:
workingContext = _store.newFormula(option_inputs[0] + "#_work")
newTopLevelFormula(workingContext)
else: # default input
if option_first_format is None: option_first_format = option_format
ContentType = {
"rdf": "application/xml+rdf",
"n3": "text/n3",
"sparql": "x-application/sparql"
}[option_first_format]
workingContext = _store.load(
# asIfFrom = join(_baseURI, ".stdin"),
asIfFrom=_baseURI,
contentType=ContentType,
flags=option_flags[option_first_format],
remember=0,
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
workingContext.stayOpen = 1 # Never canonicalize this. Never share it.
# ____________________________________________________________________
# Take commands from command line:- - - - - P A S S 2
option_format = "n3" # Use RDF/n3 rather than RDF/XML
option_flags = {"rdf": "l", "n3": "", "think": "", "sparql": ""}
option_quiet = 0
_outURI = _baseURI
option_baseURI = _baseURI # To start with
def filterize():
"""implementation of --filter
for the --filter command, so we don't have it printed twice
"""
global workingContext
global r
workingContext = workingContext.canonicalize()
_store._formulaeOfLength = {}
filterContext = _store.newFormula()
newTopLevelFormula(filterContext)
_store.load(_uri, openFormula=filterContext, why=myReason, referer="")
_newContext = _store.newFormula()
newTopLevelFormula(_newContext)
applyRules(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
sparql_query_formula = None
for arg in sys.argv[1:]: # Command line options after script name
if verbosity() > 5: progress("Processing %s." % (arg))
if arg.startswith("--"): arg = arg[1:] # Chop posix-style -- to -
_equals = string.find(arg, "=")
_lhs = ""
_rhs = ""
if _equals >= 0:
_lhs = arg[:_equals]
_rhs = arg[_equals + 1:]
try:
_uri = join(option_baseURI, _rhs)
except ValueError:
_uri = _rhs
if arg[0] != "-":
_inputURI = join(option_baseURI, splitFrag(arg)[0])
assert ':' in _inputURI
ContentType = {
"rdf": "application/xml+rdf",
"n3": "text/n3",
"sparql": "x-application/sparql"
}[option_format]
if not option_pipe: workingContext.reopen()
try:
load(_store,
_inputURI,
openFormula=workingContext,
contentType=ContentType,
flags=option_flags[option_format],
referer="",
why=myReason)
except:
progress(_inputURI)
raise
_gotInput = 1
elif arg == "-help":
pass # shouldn't happen
elif arg == "-revision":
pass
elif _lhs == "-base":
option_baseURI = _uri
if verbosity() > 10: progress("Base now " + option_baseURI)
elif arg == "-ugly":
option_outputStyle = arg
elif arg == "-crypto":
pass
elif arg == "-pipe":
pass
elif _lhs == "-outURI":
option_outURI = _uri
elif arg == "-rdf":
option_format = "rdf"
elif _lhs == "-rdf":
option_format = "rdf"
option_flags["rdf"] = _rhs
elif _lhs == "-mode":
option_flags["think"] = _rhs
elif _lhs == "-closure":
workingContext.setClosureMode(_rhs)
elif arg == "-n3":
option_format = "n3"
elif _lhs == "-n3":
option_format = "n3"
option_flags["n3"] = _rhs
elif _lhs == "-language":
option_format = _rhs
if option_first_format == None:
option_first_format = option_format
elif _lhs == "-languageOptions":
option_flags[option_format] = _lhs
elif arg == "-quiet":
option_quiet = 1
elif _lhs == "-chatty":
setVerbosity(int(_rhs))
elif arg[:7] == "-track=":
diag.tracking = int(_rhs)
elif option_pipe: ############## End of pipable options
print "# Command line error: %s illegal option with -pipe", arg
break
elif arg == "-triples" or arg == "-ntriples":
option_format = "n3"
option_flags["n3"] = "spartan"
option_outputStyle = "-bySubject"
option_quiet = 1
elif arg == "-bySubject":
option_outputStyle = arg
elif arg == "-debugString":
option_outputStyle = arg
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext)
elif arg[:7] == "-apply=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
applyRules(workingContext, filterContext)
elif arg[:7] == "-patch=":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
workingContext.reopen()
patch(workingContext, filterContext)
elif _lhs == "-filter":
filterize()
elif _lhs == "-query":
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
flags=option_flags[option_format],
referer="",
why=myReason,
topLevel=True)
_newContext = _store.newFormula()
applyQueries(workingContext, filterContext, _newContext)
workingContext.close()
workingContext = _newContext
elif _lhs == "-sparql":
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
filterContext = _store.load(_uri,
why=myReason,
referer="",
contentType="x-application/sparql")
_newContext = _store.newFormula()
_newContext.stayOpen = True
sparql_query_formula = filterContext
applySparqlQueries(workingContext, filterContext, _newContext)
# workingContext.close()
workingContext = _newContext
elif _lhs == "-why" or arg == "-why":
workingContext.stayOpen = False
workingContext = workingContext.close()
workingContext = explainFormula(workingContext, option_why)
# Can't prove proofs
diag.tracking = 0
diag.setTracking(0)
elif arg == "-dump":
workingContext = workingContext.canonicalize()
progress("\nDump of working formula:\n" +
workingContext.debugString())
elif arg == "-purge":
workingContext.reopen()
_store.purge(workingContext)
elif arg == "-purge-rules" or arg == "-data":
workingContext.reopen()
_store.purgeExceptData(workingContext)
elif arg == "-rules":
workingContext.reopen()
applyRules(workingContext, workingContext)
elif arg[:7] == "-think=":
filterContext = _store.load(_uri,
referer="",
why=myReason,
topLevel=True)
if verbosity() > 4:
progress("Input rules to --think from " + _uri)
workingContext.reopen()
think(workingContext, filterContext, mode=option_flags["think"])
elif arg[:7] == "-solve=":
# --solve is a combination of --think and --filter.
think(workingContext, mode=option_flags["think"])
filterize()
elif _lhs == "-engine":
option_engine = _rhs
elif arg == "-think":
workingContext.isWorkingContext = True
think(workingContext, mode=option_flags["think"])
elif arg == '-rete':
from swap import pycwmko
pythink = pycwmko.directPychinkoQuery(workingContext)
#return
#pythink()
"""
from pychinko import interpreter
from swap.set_importer import Set, ImmutableSet
pyf = pycwmko.N3Loader.N3Loader()
conv = pycwmko.ToPyStore(pyf)
conv.statements(workingContext)
interp = interpreter.Interpreter(pyf.rules[:])
interp.addFacts(Set(pyf.facts), initialSet=True)
interp.run()
pyf.facts = interp.totalFacts
workingContext = workingContext.store.newFormula()
reconv = pycwmko.FromPyStore(workingContext, pyf)
reconv.run()
"""
elif arg == '-sparqlServer':
from swap.sparql import webserver
from swap import cwm_sparql
sandBoxed(True)
workingContext.stayOpen = False
workingContext = workingContext.canonicalize()
def _handler(s):
return cwm_sparql.sparql_queryString(workingContext, s)
webserver.sparql_handler = _handler
webserver.run()
elif arg == "-lxkbdump": # just for debugging
raise NotImplementedError
elif arg == "-lxfdump": # just for debugging
raise NotImplementedError
elif _lhs == "-prove":
# code copied from -filter without really being understood -sdh
_tmpstore = llyn.RDFStore(_outURI + "#_g",
metaURI=_metaURI,
argv=option_with,
crypto=option_crypto)
tmpContext = _tmpstore.newFormula(_uri + "#_formula")
_newURI = join(_baseURI, "_w_" + ` _genid `) # Intermediate
_genid = _genid + 1
_newContext = _tmpstore.newFormula(_newURI + "#_formula")
_tmpstore.loadURI(_uri)
print targetkb
elif arg == "-flatten":
#raise NotImplementedError
from swap import reify
workingContext = reify.flatten(workingContext)
elif arg == "-unflatten":
from swap import reify
workingContext = reify.unflatten(workingContext)
#raise NotImplementedError
elif arg == "-reify":
from swap import reify
workingContext = reify.reify(workingContext)
elif arg == "-dereify":
from swap import reify
workingContext = reify.dereify(workingContext)
elif arg == "-size":
progress("Size: %i statements in store, %i in working formula." %
(_store.size, workingContext.size()))
elif arg == "-strings": # suppress output
workingContext.outputStrings()
option_outputStyle = "-no"
elif arg == '-sparqlResults':
from cwm_sparql import outputString, SPARQL_NS
ns = _store.newSymbol(SPARQL_NS)
if not sparql_query_formula:
raise ValueError('No query')
else:
sys.stdout.write(
outputString(sparql_query_formula,
workingContext)[0].encode('utf_8'))
option_outputStyle = "-no"
elif arg == "-no": # suppress output
option_outputStyle = arg
elif arg[:8] == "-outURI=":
pass
elif arg == "-with":
break
else:
progress("cwm: Unknown option: " + arg)
sys.exit(-1)
# Squirt it out if not piped
workingContext.stayOpen = 0 # End its use as an always-open knoweldge base
if option_pipe:
workingContext.endDoc()
else:
if hasattr(_outSink, "serializeKB"):
raise NotImplementedError
else:
if verbosity() > 5: progress("Begining output.")
workingContext = workingContext.close()
assert workingContext.canonical != None
if option_outputStyle == "-ugly":
_store.dumpChronological(workingContext, _outSink)
elif option_outputStyle == "-bySubject":
_store.dumpBySubject(workingContext, _outSink)
elif option_outputStyle == "-no":
pass
elif option_outputStyle == "-debugString":
print workingContext.debugString()
else: # "-best"
_store.dumpNested(workingContext,
_outSink,
flags=option_flags[option_format])
OFX = Namespace('http://www.w3.org/2000/10/swap/pim/ofx#')
qu = Namespace("http://www.w3.org/2000/10/swap/pim/qif#")
rdf = Namespace(RDF_NS_URI)
rdfs = Namespace("http://www.w3.org/2000/01/rdf-schema#")
cur = Namespace("http://www.w3.org/2007/ont/currency#")
# cat = Namespace("categories.n3#")
info = lambda s: sys.stderr.write(s + '\n')
monthName = [
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct",
"Nov", "Dec"
]
currencySource = uripath.join(uripath.base(), "currencies.n3")
kb = None
def figureBalances(startDate, endDate, inputURIs=["/dev/stdin"]):
global verbose
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# Load the data:
def doCommand(serialDevice=None,
outputURI=None,
doTracks=1,
doWaypoints=1,
verbose=0):
if os.name == 'nt':
if not serialDevice: serialDevice = "com1"
phys = Win32SerialLink(serialDevice)
else:
if not serialDevice: serialDevice = "/dev/ttyS0"
# serialDevice = "/dev/cu.USA19H191P1.1"
phys = UnixSerialLink(serialDevice)
gps = Garmin(phys)
print "GPS Product ID: %d Descriptions: %s Software version: %2.2f" % \
(gps.prod_id, gps.prod_descs, gps.soft_ver)
f = formula() # Empty store of RDF data
base = uripath.base()
record = f.newBlankNode()
f.add(record, RDF.type, GPS.Record)
if doWaypoints:
# show waypoints
if verbose: print "Getting waypoints"
wpts = gps.getWaypoints()
for w in wpts:
if verbose: progress( ` w `)
wpt = symbol(uripath.join(base, w.ident))
f.add(record, GPS.waypoint, wpt)
f.add(wpt, WGS.lat, obj=intern(degrees(w.slat)))
f.add(wpt, WGS.long, obj=intern(degrees(w.slon)))
if doTracks:
# show track
if verbose: print "Getting tracks"
tracks = gps.getTracks()
for t in tracks:
track = f.newBlankNode()
f.add(record, GPS.track, track)
for p in t:
if isinstance(p, TrackHdr):
if verbose: progress( ` p `)
f.add(track, GPS.disp, intern(p.dspl))
f.add(track, GPS.color, intern(p.color))
f.add(track, GPS.trk_ident, intern(p.trk_ident))
else:
if verbose: progress( ` p `)
point = f.newBlankNode()
f.add(track, GPS.trackpoint, point)
f.add(point, WGS.lat, obj=intern(degrees(p.slat)))
f.add(point, WGS.long, obj=intern(degrees(p.slon)))
# if verbose: progress(" time=", p.time)
# progress('p.time='+`p.time`) # @@
if p.time == 0 or p.time == 0xffffffffL:
if verbose: progress("time=%8x, ignoring" % p.time)
else:
f.add(point,
WGS.time,
obj=intern(isodate.fullString(TimeEpoch +
p.time)))
phys.f.close() # Should really be done by the del() below, but isn't
del (phys) # close serial link (?)
f = f.close()
if verbose:
progress("Beginning output. You can disconnect the GPS now.")
s = f.n3String(
base=base,
flags="d") # Flag - no default prefix, preserve gps: prefix hint
if outputURI != None:
op = open(outputURI, "w")
op.write(s)
op.close()
else:
print s
'file:///devel/dig/': 'http://dig.csail.mit.edu/'}
service = 'http://pingthesemanticweb.com/rest/?url='
import os, sys # http://pydoc.org/1.6/os.html
from urllib2 import urlopen
import swap
from swap import uripath # http://www.w3.org/2000/10/swap/uripath.py
from swap.uripath import join, URI_unreserved
cwd = os.getcwd()
base = 'file://' + cwd + "/"
for arg in sys.argv[1:]:
uri = join(base, arg)
for local, web in map.items():
uri = uri.replace(local, web)
if uri.endswith('.rdf'): uri = uri[:-4]
if uri.endswith('.n3'): uri = uri[:-3]
print "Ping: ",uri
encoded = ""
for ch in uri:
if ch in URI_unreserved: encoded += ch
else: encoded += "%%%2x" % ord(ch)
# print " with ", service+encoded
x = urlopen(service+encoded)
buf = x.read()
if buf.find('Thanks for') < 0: print buf
x.close()
from math import log, exp
OFX = Namespace('http://www.w3.org/2000/10/swap/pim/ofx#');
qu = Namespace("http://www.w3.org/2000/10/swap/pim/qif#")
rdf = Namespace(RDF_NS_URI)
rdfs = Namespace("http://www.w3.org/2000/01/rdf-schema#")
cur = Namespace("http://www.w3.org/2007/ont/currency#")
# cat = Namespace("categories.n3#")
info = lambda s: sys.stderr.write(s+'\n');
monthName= ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul",
"Aug", "Sep", "Oct", "Nov", "Dec"]
currencySource = uripath.join(uripath.base(), "currencies.n3");
kb = None;
def figureBalances(startDate, endDate, inputURIs=["/dev/stdin"]):
global verbose
# The base URI for this process - the Web equiv of cwd
_baseURI = uripath.base()
_outURI = _baseURI
option_baseURI = _baseURI # To start with - then tracks running base
# Load the data:
