def update(self, queryfunc, changes, subpath=None) :
"""Updates the database by running the query and then running
each of the instructions in 'changes'. The update can be
restricted to a portion of the database using the 'subpath'
argument.
The 'changes' argument is a list of path/value pairs. Each
value is run with the variable of the queryfunc bound to one
of the results of running the queryfunc on the database, and
that path in the object is updated to the result of the
value."""
queryfunc = util.assert_type(queryfunc, queries.Func)
self.lock.write_lock.acquire()
try :
data = self.data
if subpath is not None and assert_type(subpath, queries.Path) :
data = subpath.get(data)
queries.update(data, queryfunc, changes)
except queries.InconsistentData :
self.rollback()
raise
except :
# no changes made
raise
else :
self.lock.read_lock.acquire()
finally :
self.lock.write_lock.release()
self.commit()
self.lock.read_lock.release()
def __init__(self, path, valuefunc, append=False, newkey=False):
if append and newkey:
raise Exception("Not both 'append' and 'newkey' can be True")
self.path = assert_type(path, Path)
self.valuefunc = assert_type(valuefunc, ValueFunc)
self.append = append
self.newkey = newkey
def select(self, queryfunc, subpath=None) :
"""Returns the results of the query function when given the
database. The database can be restricted using the 'subpath'
argument."""
queryfunc = util.assert_type(queryfunc, queries.Func)
with self.lock.read_lock :
data = self.data
if subpath is not None and assert_type(subpath, queries.Path) :
data = subpath.get(data)
return queries.select(data, queryfunc)
def deserialize(bits):
assert_type(bits, BitArray, 'deserialize')
s = bits.tobytes()
assert_type(s, str, 'deserialize')
# print 'serialized payload from twitter: %s bytes -> %s bytes' % \
# (len(bits) / 8.0, len(s))
print 'serialized payload from twitter: %s bytes' % len(s)
x = bson.loads(s)
if not is_file(x) and not is_dir(x):
raise ArgumentError('FATAL: bad type "%s"' % x['type'])
return x
def insert(self, path, o, append=False, overwrite=False, subpath=None) :
"""Insert an object into a given path. The database can be
restricted using the subpath parameter.
If 'append' is true, then the destination must either be empty
or a list. Empty is taken to mean the destination is an empty
list. Then the element is appended to the list.
Otherwise, it is an error for the destination to not be empty,
unless 'overwrite' is true, in which case the destination is
overwritten.
The database is committed to disk on success."""
if not check_type_is_ok(o) :
raise TypeError("Object contains database-unfriendly type.")
with self.lock.write_lock :
attachmentPoint = self.data
if subpath is not None and assert_type(subpath, queries.Path) :
attachmentPoint = subpath.get(attachmentPoint)
if path is None or (path.parent is None and path.key is None):
raise Exception("Cannot insert an object with None path")
elif path.parent is not None :
attachmentPoint = path.parent.get(attachmentPoint)
if append :
attachmentPoint = attachmentPoint.setdefault(path.key, [])
if type(attachmentPoint) is not list :
raise Exception("Cannot append to non-list")
attachmentPoint.append(o)
else :
if path.key in attachmentPoint and not overwrite :
raise Exception("Cannot insert object over another object")
attachmentPoint[path.key] = o
self.commit()
def reveal(self, ciphertext, do_test=True):
assert_type(ciphertext, str, 'reveal input')
cleartext = PS.model.cipher(self.model,
self.context,
ciphertext.split(),
'decipher')
assert_type(cleartext, BitArray, 'reveal output')
if do_test:
sink = open('/dev/null', 'w')
out, err = sys.stdout, sys.stderr
sys.stdout, sys.stderr = sink, sink
test = self.conceal(cleartext, do_test=False)
sys.stdout, sys.stderr = out, err
sink.close()
if test != ciphertext:
raise RuntimeError('reveal+conceal did not produce expected output')
print 'revealed ciphertext, output is %s bytes' % len(cleartext)
return cleartext
def unpack(payload, tweet_id, downloader, concealer, name_override=None, recur=False):
assert_type(payload, dict, 'unpack')
if is_file(payload):
data, name, perms = payload['data'], payload['name'], int(payload['perms'])
if name_override:
name = name_override
fatal_if_exists(name, 'file')
print 'unpacked "%s" from tweet_id %s' % (name, tweet_id)
write_file(name, data)
print 'permissions: %s' % perms
chmod(name, perms)
print ''
elif is_dir(payload):
ids, name, perms = payload['ids'], payload['name'], int(payload['perms'])
if name_override:
name = name_override
fatal_if_exists(name, 'directory')
print 'unpacked "%s" with child tweet_ids %s' % (name, ids)
write_dir(name)
print 'permissions: %s' % perms
chmod(name, perms)
print ''
if recur:
chdir(name)
for tweet_id in ids:
payload = downloader(tweet_id)
payload = concealer.reveal(payload)
payload = deserialize(payload)
unpack(payload,
tweet_id,
downloader,
concealer,
name_override=None,
recur=recur)
chdir('..')
def save_model(hyperparams, model, get_filename):
'''
hyperparams : dict of hyper parameters
model : keras Model or pytorch Module
get_filename : a function/or lambda that takes in a filename and retuns saveable path
'''
util.assert_type(hyperparams, dict)
util.assert_type(model, Module)
assert callable(
get_filename), 'takes in a filename and retuns saveable path'
with open(get_filename('hyperparameters.json'), 'w') as f:
json.dump(hyperparams, f, sort_keys=True, indent=2)
with open(get_filename('model.json'), 'w') as f:
f.write(model.to_json(indent=2))
stdout = sys.stdout
with open(get_filename('summary.txt'), 'w') as sys.stdout:
if isinstance(model, Module):
sys.stdout.write(str(model))
sys.stdout = stdout
return
def remove(self, queryfunc, subpath=None) :
"""Remove from the database all entries returned by the given
query function when applied to the database. The database can
be restricted using the 'subpath' parameter.
The database is committed to disk on success."""
self.lock.write_lock.acquire()
try :
data = self.data
if subpath is not None and assert_type(subpath, queries.Path) :
data = subpath.get(data)
queries.remove(data, queryfunc)
except queries.InconsistentData :
self.rollback()
raise
except :
# no changes made
raise
else :
self.lock.read_lock.acquire()
finally :
self.lock.write_lock.release()
self.commit()
self.lock.read_lock.release()
def __init__(self, name, *params):
if name not in util.allowed_operations:
raise Exception("Operation for Op must be allowed, not " + name)
self.name = name
self.op = util.allowed_operations[name]
self.params = [assert_type(p, Value) for p in params]
def __init__(self, value, func):
self.value = assert_type(value, Value)
self.func = assert_type(func, ValueFunc)
def __init__(self, var, query):
self.var = var
if isinstance(var, Var):
self.var = var.name
self.query = assert_type(query, Query)
def __init__(self, source, *pathparts):
self.source = assert_type(source, Value)
if len(pathparts) == 1 and isinstance(pathparts[0], Path):
self.path = pathparts[0]
else:
self.path = path(*pathparts)
def __init__(self, *params):
self.params = [assert_type(p, Value) for p in params]
def __init__(self, query, func):
self.query = assert_type(query, Query)
self.func = assert_type(func, Func)
def __init__(self, var, value):
self.var = var
if isinstance(var, Var):
self.var = var.name
self.value = assert_type(value, Value)
def __init__(self, query):
self.query = assert_type(query, Query)
def __init__(self, name):
self.name = assert_type(name, basestring)
def __init__(self, value):
self.value = assert_type(value, Value)
def __init__(self, *queries):
self.queries = [assert_type(q, Query) for q in queries]
