def encode(self, cmd):
e = "%s%s" % (cmd.cmd, self.EOL)
if self.debug > 5:
CPL.log("RawEncoder", "encoded: %s" % (e))
return e
def queueForOutput(self, s, timer=None):
""" Append s to the output queue. """
assert s != None, "queueing nothing!"
self.queueLock.acquire(src='queueForOutput')
try:
mustRegister = (self.outQueue == [])
# Keep the output "lines" separate.
#
self.outQueue.append(s)
# Add any timer.
if timer != None:
self.addTimer(timer)
# Bump the stats.
self.totalQueued += 1
if len(self.outQueue) > self.maxQueue:
self.maxQueue = len(self.outQueue)
if self.debug > 4:
CPL.log("IOHandler.queueForOutput",
"appended %r to queue (len=%d) of %s" % \
(s, len(self.outQueue), self))
if mustRegister:
self.poller.addOutput(self)
finally:
self.queueLock.release(src='queueForOutput')
def parseKVs(self, kvl):
""" Convert some form of keys to an OrderedDict.
We are trying to be ridiculously flexible here. Take:
- a string, which we parse as it came from an ICC.
- a list, which we parse either as a list of key=value strings or of (key, value) duples.
"""
if isinstance(kvl, str):
return Parsing.parseKVs(kvl)
od = collections.OrderedDict()
if kvl is not None:
for i in kvl:
if isinstance(i, str):
k, v, junk = Parsing.parseKV(i)
od[k] = v
elif type(i) in (list, tuple) and len(i) == 2:
k, v, junk = Parsing.parseKV("%s=%s" % i)
else:
CPL.log('Reply', 'kvl item is not a string: %r' % (i))
raise Exception("kvl == %r" % (i))
return od
def __init__(self, poller, **argv):
CPL.Object.__init__(self, **argv)
self.poller = poller
CPL.log("IOHandler.init", "IOHandler(argv=%s)" % (argv))
# The IO size tweaks would mean something for slow network links.
#
self.tryToRead = argv.get('readSize', 4096)
self.tryToWrite = argv.get('writeSize', 4096)
self.tryToWriteMany = argv.get('writeMany', False)
self.oneAtATime = argv.get('oneAtATime', False)
self.in_f = self.out_f = None
self.in_fd = self.out_fd = None
self.outQueue = []
self.queueLock = CPL.LLock(debug = (argv.get('debug', 0) > 7))
self.setInputFile(argv.get('in_f', None))
self.setOutputFile(argv.get('out_f', None))
# Some stats
#
self.totalReads = 0
self.totalBytesRead = 0
self.largestRead = 0
self.totalQueued = 0
self.maxQueue = 0
self.totalOutputs = 0
self.totalWrites = 0
self.totalBytesWritten = 0
self.largestWrite = 0
def reply(self, r):
# The authentication system want to be able to block all output until
# the connection is established. Let an external agent "intercept" replies.
#
intercepted = False
if hasattr(self, 'interceptReply'):
intercepted = self.interceptReply(r)
if intercepted:
return
# Most replies get sent to all interested commanders. But we allow
# the possibility of only sending to the commander; in that case,
# the commander gets all replies and keys, but other commanders only get told
# about command completion.
# We do this by triaging out replies here, then optionally telling the encoder
# whether to include keys.
#
if r.bcast or r.cmd.cmdrID == self.ID:
er = self.encoder.encode(r, self)
self.queueForOutput(er)
if self.log:
self.log.log(er, note='>')
else:
CPL.log("CommanderNub.reply", "not bcast; rID=%s selfID=%s" % (r.cmd.cmdrID, self.ID))
if r.finishesCommand():
er = self.encoder.encode(r, self, noKeys=True)
self.queueForOutput(er)
if self.log:
self.log.log(er, note='>')
def setOutputFile(self, f):
""" Change the output file. Close and unregister any old file. Clear the output queue.
This should be the only method which adjusts output registration.
"""
if self.debug > 2:
CPL.log("IOHandler.setOutput", "%s changing output %s to %s. queue=%s" % \
(self, self.out_f, f, self.outQueue))
if self.out_f != None:
self.poller.removeOutput(self)
if f != self.out_f:
try:
self.out_f.close()
except:
CPL.error("IOHandler.setOutput", "failed to close output for %s", self)
# Establish new .out_f
#
self.out_f = f
if f == None:
self.out_fd = None
else:
self.out_fd = f.fileno()
self.outQueue = []
def encode(self, cmd):
if self.useCID:
if self.CIDfirst:
ids = "%s %s " % (cmd.actorCid, cmd.actorMid)
else:
ids = "%s %s " % (cmd.actorMid, cmd.actorCid)
else:
ids = "%s " % (cmd.actorMid,)
if self.sendCmdrCID:
cmdrInfo = "%s " % (cmd.cmdrCid)
elif self.sendCmdr:
cmdrInfo = "%s " % (cmd.cmdrName)
else:
cmdrInfo = ""
if self.useTarget:
e = "%s%s %s%s%s" % (cmdrInfo, cmd.actorName, ids, cmd.cmd, self.EOL)
else:
e = "%s%s%s%s" % (cmdrInfo, ids, cmd.cmd, self.EOL)
if self.debug > 5:
CPL.log("ASCIIEncoder", "encoded: %s" % (e))
return e
def genAuthKeys(self, programs=None, cmd=None):
""" Generate keys describing some or all programs.
Args:
cmd - if set, the command to reply to. Otherwise use our .defaultCmd
programs - if set, a list of programs to generate keys for. Otherwise describe all programs.
Notes:
Never finishes the cmd.
"""
if not cmd:
cmd = self.defaultCmd
if not programs:
programs = list(self.programs.keys())
programs.sort()
CPL.log("auth.genAuthKeys", "listing programs: %s" % (programs))
for prog in programs:
try:
pAuth = list(self.programs[prog].keys())
except KeyError as e:
raise Exception("No authorization entry found for program %s" % (prog))
pAuth.sort()
actors = [CPL.qstr(x) for x in pAuth]
cmd.inform("authList=%s,%s" % (CPL.qstr(prog), ','.join(actors)))
def setActorsForProgram(self, program, actors, cmd=None):
""" Define the list of actors that a program can command.
"""
if self.debug > 3:
CPL.log("auth.setActors", "setting actors for commander %s: %s" % (program, actors))
if not cmd:
cmd = self.defaultCmd
if program not in self.programs:
cmd.fail("permsTxt=%s" % (CPL.qstr("Program %s did not have an authorization entry, so could not be set" % (program))))
return
d = {}
for a in actors:
if a not in self.actors:
cmd.warn("permsTxt=%s" % (CPL.qstr("Actor %s is not subject to permissions." % (a))))
else:
d[a] = True
# Make sure God-like commanders can always command us...
#
if program in self.gods:
d['perms'] = True
self.programs[program] = d
self.genAuthKeys(programs=[program], cmd=cmd)
def dropPrograms(self, programs=[], cmd=None):
""" Remove a list of programs from the control list.
Args:
programs - a list of programs to register for authorization. Adds all connected
programs if empty.
cmd - The command to reply to, or .defaultCmd
"""
if not programs:
programs = list(self.programs.keys())
if self.debug > 3:
CPL.log("auth.dropProgram", "dropping programs %s" % (programs))
if not cmd:
cmd = self.defaultCmd
for program in programs:
if program in self.gods:
cmd.warn("permsTxt=%s" % \
(CPL.qstr("Super-%s cannot be removed from the list of authorized programs" % (program))))
self.genAuthKeys([program], cmd)
continue
try:
del self.programs[program]
except:
cmd.warn("permsTxt=%s" % \
(CPL.qstr("Program %s did not have an authorization entry, so could not be deleted" % (program))))
self.genProgramsKey(cmd=cmd)
def addPrograms(self, programs=[], actors=[], cmd=None):
""" Add a list program to the control list.
Args:
program - a list of program names. Use all connected programs if empty.
actors - an optional list of actors that the commander can command.
"""
if not cmd:
cmd = self.defaultCmd
if not programs:
programs = []
for name, cmdr in g.commanders.items():
if cmdr.needsAuth and name not in self.programs:
programs.append(name)
if self.debug > 3:
CPL.log("auth.addPrograms",
"adding programs %s with actors=%s" % (programs, actors))
for prog in programs:
if prog in self.programs:
cmd.warn("permsTxt=%s" % \
(CPL.qstr("Program %s already has an authorization entry, which will not be modified." % (prog))))
continue
self.programs[prog] = {}
self.setActorsForProgram(prog, actors, cmd=cmd)
self.genProgramsKey(cmd=cmd)
def connected(self):
""" Optionally send a list of commands after the connection has been established.
This is complicated by .grabCID. If that exists, we want to sent a command just to
force the actor to generate our CID. If we send the init comands before that is
established, we will not recognize the replies as replies to our init commands.
So if .grabCID is set and our .cid is not, send the .grabCID command, and let the .grabCID
logic in copeWithInput() call us back when we are _really_ connected. Feh.
"""
if self.grabCID != False and self.cid == None:
if isinstance(self.grabCID, basestring):
initCmds = [self.grabCID]
else:
initCmds = []
doRegister = False
else:
initCmds = self.initCmds
doRegister = True
CPL.log("ActorNub.connected", "sending initCmds to %s (cid=%s)" % (self.ID, self.cid))
for c in initCmds:
CPL.log("ActorNub.connected", "sending initCmd %s" % (c))
self.sendCommand(Command('.hub', '0',
g.hubMIDs.gimme(),
self.name, c),
doRegister=doRegister)
def lockActors(self, actors, cmd=None):
""" Block non-APO users form commanding a list of actors.
Any actors not in .actors will be ignored with a warning. This may not be the right behavior. We may
actually want to allow actors we do not control to be locked.
"""
if not cmd:
cmd = self.defaultCmd
if not actors:
actors = self.actors
if self.debug > 3:
CPL.log("auth.lockActor", "locking actors %s" % (actors))
for a in actors:
if a in self.lockedActors:
cmd.warn("permsTxt=%s" % (CPL.qstr("Actor %s is already locked" % (a))))
elif a not in self.actors:
cmd.warn("permsTxt=%s" % (CPL.qstr("Actor %s is not subject to permissions and will not be locked" % (a))))
else:
self.lockedActors[a] = True
self.genLockedKey(cmd=cmd)
def ioshutdown(self, **argv):
""" Unregister ourselves """
why = argv.get('why', "just cuz")
CPL.log("IOhandler.ioshutdown", "what=%s why=%s" % (self, why))
self.setOutputFile(None)
self.setInputFile(None)
def addSource(self, source):
""" Register the fact that a given source exists. """
CPL.log("KVDict.addSource", "adding source %s" % (source))
if source in self.sources:
CPL.log("KVDict.addSource", "source %s already exists" % (source))
return
self.sources[source] = cdict(dictType=collections.OrderedDict)
def acquire(self, block=True, src="up"):
if self.debug > 0:
CPL.log("LLock.acquiring", "name=%s, block=%s, src=%s" % \
(self.name, block, src))
self.lock.acquire(block)
if self.debug > 0:
CPL.log("LLock.acquired", "name=%s, block=%s, src=%s" % \
(self.name, block, src))
def setNames(self, programName, username):
""" Set our program and usernames. """
CPL.log('CommandeNub.setNames', 'setting name for %s to %s.%s' % (self, programName, username))
newName = hub.validateCommanderNames(self, programName, username)
self.taster.removeFromFilter([], [self.name], [self.name])
self.taster.addToFilter([], [newName], [newName])
self.setName(newName)
def decode(self, buf, newData):
""" Find and extract a single complete reply in the buf. Uses .EOL to
recognize the end of a reply.
Returns:
- a Reply instance. None if .EOL no found in buf.
- the content of buf with the first complete reply removed.
Always consumes input up to the first .EOL, if .EOL is found.
If .EOL is found, but the input can not be properly parsed, a modified reply is generated:
- If no header information is found (i.e. no MID, CID, etc), the following is returned:
w RawInput="full line"
- If header information is found, but some part of the keywords are not parseable,
x K1=V1; K2; RawKeys="rest of line"
- If header information is found, and the last valueis an unterminated string, that
value is silently terminated.
"""
if newData:
buf += newData
if self.debug > 5:
CPL.log('Stdin.extractReply', "called with EOL=%r and buf=%r" % (self.EOL, buf))
eol = buf.find(self.EOL)
if self.debug > 4:
CPL.log('Stdin.extractReply', "eol at %d in buffer %r" % (eol, buf))
# No complete reply found. make sure to return
# the unmolested buffer.
#
if eol == -1:
return None, buf
replyString = buf[:eol]
buf = buf[eol+len(self.EOL):]
if self.debug > 2:
CPL.log('Stdin.extractReply', "hoping to parse (CIDfirst=%s) %r" % (self.cidFirst, replyString))
for c in self.stripChars:
replyString = replyString.replace(c, '')
# Make sure to consume unparseable junk up to the next EOL.
#
try:
r = parseASCIIReply(replyString, cidFirst=self.cidFirst)
except SyntaxError as e:
CPL.log("ASCIIReplyDecoder", "Parsing error from %s: %r" % (self.name, e))
return None, buf
if self.debug > 3:
CPL.log('Stdin.extractReply', "extracted %r, returning %r" % (r, buf))
return r, buf
def reportQueued(self):
if g.hubcmd != None and self.bcastCmdInfo:
dcmd = self.cmd if len(self.cmd) < 1000 else self.cmd[:1000] + "...."
g.hubcmd.diag(("CmdQueued=%d,%0.2f,%s,%s,%s,%s,%s" %
(self.xid, self.ctime,
CPL.qstr(self.cmdrCid), self.cmdrMid,
CPL.qstr(self.actorName), self.actorMid,
CPL.qstr(dcmd))),
src='cmds')
def __init__(self, debug=0, name=None):
self.debug = debug
self.lock = Lock()
if name == None:
name = 'lock-%04d' % (self.seq)
self.seq += 1
self.name = name
if self.debug > 0:
CPL.log("LLock.create", "name=%s" % (self.name))
def readInput(self):
""" Pseudo-callback used to make the poller reconfigure itself. Does not need
to actually do anything.
"""
if self.debug > 0:
CPL.log('NullIO', 'reading token')
d = os.read(self.fd, 1)
def listCommandsCmd(self, cmd, doFinish=True):
""" Send command keywords.
"""
cmd.inform('actorCmds=%s,%d,%d' % \
(CPL.qstr(self.name), len(self.liveCommands), len(self.ourCommands)))
for id, ourCmd in self.ourCommands.items():
cmd.inform('actorCmd=%s,%s,%s' % \
(CPL.qstr(self.name), CPL.qstr(id), CPL.qstr(ourCmd)))
def getKey(self, src, key, default=None):
if self.debug > 3:
CPL.log("KVDict.getKey", "get src=%s key=%s" % (src, key))
if src not in self.sources:
return default
val = self.sources[src].get(key, default)
if val == default:
return val
return val.val
def makeAndSendReply(self, flag, KVs, **argv):
""" Bundle a flag and some KVs into a proper Reply & ship it off. """
src = argv.get('src', self.actorName)
bcast = argv.get('bcast', True)
if self.debug > 0:
CPL.log("Command.makeAndSendReply", "src = %r, flag = %s, KVs = %r" % (src, flag, KVs))
r = Reply(self, flag, KVs, src=src, bcast=bcast)
self.reply(r, **argv)
def __init__(self, **argv):
CommandDecoder.__init__(self, **argv)
self.EOL = argv.get('EOL', '\n')
self.needCID = argv.get('needCID', True)
self.needMID = argv.get('needMID', True)
self.CIDfirst = argv.get('CIDfirst', False)
if self.needCID and not self.needMID:
CPL.log("ASCIICmdDecoder", "if CID is needed, than MID must also be.")
if self.needMID == False:
self.mid = 1
def __registerCmd(self, cmd, ours):
""" """
key = self.keyForCommand(cmd)
if self.debug > 0:
CPL.log("Nub", "registering key(ours=%s)=%s for %s" % (key, ours, cmd))
if ours and key in self.liveCommands:
raise RuntimeError("Duplicate command key for %s: %s" % (self, key))
self.liveCommands[key] = cmd
if ours:
self.ourCommands[key] = cmd
def readInput(self):
CPL.log("IOAccept.readInput", "accepting...")
newfd, addr = self.listenFd.accept()
# Listen for a single connect. Kill ourselves if we should.
#
if self.acceptMany == 0:
self.poller.removeInput(self)
self.poller = None
self.listenFd.listen(self.depth)
if self.callback:
self.callback(newfd, addr)
def decode(self, buf, newData):
""" Find and extract a single complete command in the inputBuffer.
"""
if newData:
buf += newData
if self.debug > 3:
CPL.log('PyReply.decoder', "called with EOL=%r and buf=%r" % (self.EOL, buf))
eol = buf.find(self.EOL)
if self.debug > 2:
CPL.log('PyReply.decoder', "eol at %d in buffer %r" % (eol, buf))
# No complete reply found. make sure to return
# the unmolested buffer.
#
if eol == -1:
return None, buf
replyString = buf[:eol]
buf = buf[eol+len(self.EOL):]
# Make sure to consume unparseable junk up to the next EOL.
#
try:
r = pickle.loads(replyString)
except SyntaxError as e:
CPL.log("PyReply.decoder", "Failed to unpickle %r" % (replyString))
return None, buf
if self.debug > 5:
CPL.log('PyReply.decoder', "extracted %r, returning %r" % (r, buf))
return r, buf
def setKV(self, src, key, val, reply):
""" Save the
"""
if src == None:
src = reply.src
if self.debug > 5:
CPL.log("KVDict.setKV", "src=%r, key=%r, val=%r" % (src, key, val))
if src not in self.sources:
self.sources[src] = cdict(dictType=collections.OrderedDict)
self.sources[src][key] = KV(key, val, reply)
def encode(self, r, nub, noKeys=False):
""" Encode a protocol-free reply for a given nub. """
if self.keyName:
rawVal = r.KVs.get(self.keyName, '')
val = dequote(rawVal)
CPL.log('RAWDEQUOTE', "rawVal=%r val=%r" % (rawVal, val))
else:
val = self.encodeKeys(r.src, r.KVs)
if val:
return "%s%s" % (val, self.EOL)
else:
return ''
def objects_lookup():
content = request.get_json()
type_int = CPL.object_str_to_type(str(content['type']))
objects = connection.lookup_all_objects(
str(content['prefix']), str(content['name']), type_int,
cpl_bundle(str(content['bundle']) if content['bundle'] else None))
return jsonify(ids=[o.id for o in objects])
def serialize_object_info(obj):
return {
'id': obj.object.id,
'creation_time': obj.creation_time,
'prefix': obj.prefix,
'name': obj.name,
'type': CPL.object_type_to_str(obj.type),
'bundle': obj.id
}
def serialize_relation_info(rel):
return {
'id': rel.id,
'ancestor': rel.ancestor.id,
'descendant': rel.descendant.id,
'type': CPL.relation_type_to_str(rel.type),
'bundle': rel.id,
'base': rel.base.id,
'other': rel.other.id
}
def object_relation_post(id):
content = request.get_json()
type_int = CPL.relation_str_to_type(str(content['type']))
if content['dest']:
relation = cpl_object(id).relation_to(
cpl_object(int(content['dest'])), type_int,
cpl_bundle(int(content['bundle'])))
else:
relation = cpl_object(id).relation_from(
cpl_object(int(content['src'])), type_int,
cpl_bundle(int(content['bundle'])))
return jsonify(id=relation.id)
"""
"""
Test python bindings.
Currently, we only support ODBC, although we should add the RDF support and
tests as well.
"""
import CPL
import sys
import tempfile
import random
prefix = 'ptst'
iri = "python.test"
c = CPL.cpl_connection()
print "Session information: "
CPL.p_session(c.session)
# Create objects
print
print '----- Create object tests -----'
print
rand = '_' + str(random.randint(0, 1000000))
bundle_name = 'Bundle' + rand
print('Create bundle name:' + bundle_name)
bundle = c.create_bundle(bundle_name, prefix)
CPL.p_object(bundle)
def object_post():
content = request.get_json()
type_int = CPL.object_str_to_type(str(content['type']))
o = connection.create_object(str(content['prefix']), str(content['name']),
type_int, cpl_bundle(int(content['bundle'])))
return jsonify(id=o.id)
Contributor(s): Margo Seltzer, Peter Macko
"""
"""
Test python bindings.
Currently, we only support ODBC, although we should add the RDF support and
tests as well.
"""
import CPL
import sys
import tempfile
prefix = 'ptst'
iri = "python.test"
c = CPL.cpl_connection()
print "Session information: "
CPL.p_session(c.session)
# Create objects
print
print '----- Create object tests -----'
print
bundle_name = 'Bundle'
print('Create bundle name:' + bundle_name)
bundle = c.create_bundle(bundle_name)
CPL.p_bundle(bundle, False)
print('Add bundle prefix:' + prefix + ':' + iri)
import argparse
import CPL
import csv
import os
import shutil
import json
fileExt = ".json"
originator = 'test'
output_path = '../input'
graph_path = output_path + '/graphs/'
macro_edge_file = output_path + '/synthetic_edges.csv'
db_connection = CPL.cpl_connection()
bundle_index = 0
object_index = 0
obj_to_bundle = {}
bundle_connections = set()
def create_macro_edges_file():
with open(macro_edge_file, mode='w') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(['graph_1', 'graph_2'])
map(writer.writerow, bundle_connections)
def create_data_file(bundle, tag):
global object_index
global bundle_index
import json
import sys
#sys.path.append('C:\\Users\\johns\\Documents\\prov-cpl\\bindings\\python')
#sys.path.insert(0, 'C:\\Users\\johns\\Documents\\prov-cpl\\bindings\\python')
import CPL
from CPL import cpl_relation
originator = "demo"
c = CPL.cpl_connection()
# with open("prov.json") as f:
# data = json.loads(f.read())
# c.import_document_json(data, "testrprov", None, 0)
with open("articles.json") as f:
data = json.loads(f.read())
##################
bundle_name = str(data[0]["url"])
bundle_type = CPL.ENTITY
try:
print("about to look up")
bundle = c.lookup_bundle(bundle_name, originator)
except Exception as e:
print("about to create")
bundle = c.create_bundle(bundle_name, originator)
# CPL.p_bundle(bundle, False)
stuff = []
relations = [] # type: List[cpl_relation]
strings = []
node_names = 0 # counter
Contributor(s): Margo Seltzer, Peter Macko
"""
"""
Test python bindings.
Currently, we only support ODBC, although we should add the RDF support and
tests as well.
"""
import CPL
import sys
import tempfile
originator = 'python_test'
c = CPL.cpl_connection()
print "Session information: "
CPL.p_session(c.session)
# Create objects
print
print '----- Create tests -----'
print
obj1_name = "Process A"
obj1_type = 'proc'
print('Create object name: ' + obj1_name + ' type: ' + obj1_type +
' container: void')
o1 = c.create_object(originator, obj1_name, obj1_type)
CPL.p_object(o1, True)