def update(self, queryfunc, changes, subpath=None) : """Updates the database by running the query and then running each of the instructions in 'changes'. The update can be restricted to a portion of the database using the 'subpath' argument. The 'changes' argument is a list of path/value pairs. Each value is run with the variable of the queryfunc bound to one of the results of running the queryfunc on the database, and that path in the object is updated to the result of the value.""" queryfunc = util.assert_type(queryfunc, queries.Func) self.lock.write_lock.acquire() try : data = self.data if subpath is not None and assert_type(subpath, queries.Path) : data = subpath.get(data) queries.update(data, queryfunc, changes) except queries.InconsistentData : self.rollback() raise except : # no changes made raise else : self.lock.read_lock.acquire() finally : self.lock.write_lock.release() self.commit() self.lock.read_lock.release()
def __init__(self, path, valuefunc, append=False, newkey=False): if append and newkey: raise Exception("Not both 'append' and 'newkey' can be True") self.path = assert_type(path, Path) self.valuefunc = assert_type(valuefunc, ValueFunc) self.append = append self.newkey = newkey
def select(self, queryfunc, subpath=None) : """Returns the results of the query function when given the database. The database can be restricted using the 'subpath' argument.""" queryfunc = util.assert_type(queryfunc, queries.Func) with self.lock.read_lock : data = self.data if subpath is not None and assert_type(subpath, queries.Path) : data = subpath.get(data) return queries.select(data, queryfunc)
def deserialize(bits): assert_type(bits, BitArray, 'deserialize') s = bits.tobytes() assert_type(s, str, 'deserialize') # print 'serialized payload from twitter: %s bytes -> %s bytes' % \ # (len(bits) / 8.0, len(s)) print 'serialized payload from twitter: %s bytes' % len(s) x = bson.loads(s) if not is_file(x) and not is_dir(x): raise ArgumentError('FATAL: bad type "%s"' % x['type']) return x
def insert(self, path, o, append=False, overwrite=False, subpath=None) : """Insert an object into a given path. The database can be restricted using the subpath parameter. If 'append' is true, then the destination must either be empty or a list. Empty is taken to mean the destination is an empty list. Then the element is appended to the list. Otherwise, it is an error for the destination to not be empty, unless 'overwrite' is true, in which case the destination is overwritten. The database is committed to disk on success.""" if not check_type_is_ok(o) : raise TypeError("Object contains database-unfriendly type.") with self.lock.write_lock : attachmentPoint = self.data if subpath is not None and assert_type(subpath, queries.Path) : attachmentPoint = subpath.get(attachmentPoint) if path is None or (path.parent is None and path.key is None): raise Exception("Cannot insert an object with None path") elif path.parent is not None : attachmentPoint = path.parent.get(attachmentPoint) if append : attachmentPoint = attachmentPoint.setdefault(path.key, []) if type(attachmentPoint) is not list : raise Exception("Cannot append to non-list") attachmentPoint.append(o) else : if path.key in attachmentPoint and not overwrite : raise Exception("Cannot insert object over another object") attachmentPoint[path.key] = o self.commit()
def reveal(self, ciphertext, do_test=True): assert_type(ciphertext, str, 'reveal input') cleartext = PS.model.cipher(self.model, self.context, ciphertext.split(), 'decipher') assert_type(cleartext, BitArray, 'reveal output') if do_test: sink = open('/dev/null', 'w') out, err = sys.stdout, sys.stderr sys.stdout, sys.stderr = sink, sink test = self.conceal(cleartext, do_test=False) sys.stdout, sys.stderr = out, err sink.close() if test != ciphertext: raise RuntimeError('reveal+conceal did not produce expected output') print 'revealed ciphertext, output is %s bytes' % len(cleartext) return cleartext
def unpack(payload, tweet_id, downloader, concealer, name_override=None, recur=False): assert_type(payload, dict, 'unpack') if is_file(payload): data, name, perms = payload['data'], payload['name'], int(payload['perms']) if name_override: name = name_override fatal_if_exists(name, 'file') print 'unpacked "%s" from tweet_id %s' % (name, tweet_id) write_file(name, data) print 'permissions: %s' % perms chmod(name, perms) print '' elif is_dir(payload): ids, name, perms = payload['ids'], payload['name'], int(payload['perms']) if name_override: name = name_override fatal_if_exists(name, 'directory') print 'unpacked "%s" with child tweet_ids %s' % (name, ids) write_dir(name) print 'permissions: %s' % perms chmod(name, perms) print '' if recur: chdir(name) for tweet_id in ids: payload = downloader(tweet_id) payload = concealer.reveal(payload) payload = deserialize(payload) unpack(payload, tweet_id, downloader, concealer, name_override=None, recur=recur) chdir('..')
def save_model(hyperparams, model, get_filename): ''' hyperparams : dict of hyper parameters model : keras Model or pytorch Module get_filename : a function/or lambda that takes in a filename and retuns saveable path ''' util.assert_type(hyperparams, dict) util.assert_type(model, Module) assert callable( get_filename), 'takes in a filename and retuns saveable path' with open(get_filename('hyperparameters.json'), 'w') as f: json.dump(hyperparams, f, sort_keys=True, indent=2) with open(get_filename('model.json'), 'w') as f: f.write(model.to_json(indent=2)) stdout = sys.stdout with open(get_filename('summary.txt'), 'w') as sys.stdout: if isinstance(model, Module): sys.stdout.write(str(model)) sys.stdout = stdout return
def remove(self, queryfunc, subpath=None) : """Remove from the database all entries returned by the given query function when applied to the database. The database can be restricted using the 'subpath' parameter. The database is committed to disk on success.""" self.lock.write_lock.acquire() try : data = self.data if subpath is not None and assert_type(subpath, queries.Path) : data = subpath.get(data) queries.remove(data, queryfunc) except queries.InconsistentData : self.rollback() raise except : # no changes made raise else : self.lock.read_lock.acquire() finally : self.lock.write_lock.release() self.commit() self.lock.read_lock.release()
def __init__(self, name, *params): if name not in util.allowed_operations: raise Exception("Operation for Op must be allowed, not " + name) self.name = name self.op = util.allowed_operations[name] self.params = [assert_type(p, Value) for p in params]
def __init__(self, value, func): self.value = assert_type(value, Value) self.func = assert_type(func, ValueFunc)
def __init__(self, var, query): self.var = var if isinstance(var, Var): self.var = var.name self.query = assert_type(query, Query)
def __init__(self, source, *pathparts): self.source = assert_type(source, Value) if len(pathparts) == 1 and isinstance(pathparts[0], Path): self.path = pathparts[0] else: self.path = path(*pathparts)
def __init__(self, *params): self.params = [assert_type(p, Value) for p in params]
def __init__(self, query, func): self.query = assert_type(query, Query) self.func = assert_type(func, Func)
def __init__(self, var, value): self.var = var if isinstance(var, Var): self.var = var.name self.value = assert_type(value, Value)
def __init__(self, query): self.query = assert_type(query, Query)
def __init__(self, name): self.name = assert_type(name, basestring)
def __init__(self, value): self.value = assert_type(value, Value)
def __init__(self, *queries): self.queries = [assert_type(q, Query) for q in queries]