def testFilter(self):
manager = md.DataManager()
manager.thing0 = np.zeros(15)
manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile)
manager.open(name='pdzmanager',
file=self.pdzfile,
open=pdzutils.PDZManager.open)
manager.store('pdzrange pdz'.split(),
manager.pdzmanager.associatePDZ(manager.cat1['SeqNr']))
def filterFunction(manager):
return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0]
theFilter = filterFunction(manager)
expectedSeq = manager.cat1.filter(theFilter)['SeqNr']
manager.update(filterFunction, {
'cat1': 'filter',
'pdz': '__getitem__'
})
self.assertEquals(len(manager.thing0), 15)
self.assertEquals(len(manager.pdzrange), 25)
self.assertTrue((manager.cat1['SeqNr'] == expectedSeq).all())
self.assertEquals(manager.pdz.shape, (len(expectedSeq), 25))
def testLogStatement(self):
manager = md.DataManager()
manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile)
manager.open(name='pdzmanager',
file=self.pdzfile,
open=pdzutils.PDZManager.open)
manager.store('pdzrange pdz'.split(),
manager.pdzmanager.associatePDZ(manager.cat1['SeqNr']))
def filterFunction(manager):
return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0]
manager.update(filterFunction, {
'cat1': 'filter',
'pdz': '__getitem__'
})
self.assertEquals(len(manager.history), 5)
log = manager.history.log()
print log
for historyitem in manager.history:
self.assertTrue(historyitem.log() in log)
def __init__(self, data, ip, port, clientserver):
self.data = data
self._ip = ip
self._port = port
self._server = clientserver
self.datamgr = datamanager.DataManager("localhost", "liwaa", "p@ss0rd",
"testDatabase")
def testFilterHistory(self):
manager = md.DataManager()
manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile)
manager.open(name='pdzmanager',
file=self.pdzfile,
open=pdzutils.PDZManager.open)
manager.store('pdzrange pdz'.split(),
manager.pdzmanager.associatePDZ(manager.cat1['SeqNr']))
def filterFunction(manager):
return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0]
preTime = datetime.datetime.now()
manager.update(filterFunction, {
'cat1': 'filter',
'pdz': '__getitem__'
})
postTime = datetime.datetime.now()
history = manager.history[-1]
self.assertTrue(preTime <= history.time <= postTime)
self.assertEquals(history.action, 'UPDATE')
self.assertEquals(history.name, ['cat1', 'pdz'])
self.assertEquals(history.method, filterFunction)
self.assertEquals(history.methodargs, ())
self.assertEquals(history.methodkeywords, {})
printedhistory = str(history)
print printedhistory
self.assertTrue(
time.asctime(history.time.timetuple()) in printedhistory)
self.assertTrue('UPDATE' in printedhistory)
self.assertTrue('cat1' in printedhistory)
self.assertTrue('pdz' in printedhistory)
methodname = filterFunction.__name__
self.assertTrue(methodname in printedhistory)
self.assertTrue(str(filterFunction) not in printedhistory)
sourcefile = inspect.getfile(filterFunction)
self.assertTrue(sourcefile in printedhistory)
sourcefile_moddate = time.asctime(
time.localtime(os.stat(sourcefile).st_mtime))
self.assertTrue(sourcefile_moddate in printedhistory)
sourcecode = inspect.getsource(filterFunction)
self.assertTrue(sourcecode in printedhistory)
self.assertTrue(str(history.methodargs) in printedhistory)
self.assertTrue(str(history.methodkeywords) in printedhistory)
def testImmutibility(self):
manager = md.DataManager()
manager.sizecol = 'rh'
self.assertRaises(md.ManagedDataException, setattr, manager, 'sizecol',
32)
self.assertRaises(md.ManagedDataException, manager.__setitem__,
'sizecol', 42)
def load_from_manager(cls, path):
"""Create an instance of this class by loading its data from files."""
dm = datamanager.DataManager(path)
meta, data = dm.get()
inst = cls(meta['conf'], _skip_init=True)
inst.roads = geom.Segments(data['roads'])
inst.translation = data['translation']
inst.scaling = data['scaling']
return inst
def __init__(self):
super(ApplicationWindow, self).__init__()
QtWidgets.QMainWindow.__init__(self)
Ui_MainWindow.__init__(self)
self.setupUi(self)
self.dataManager = dm.DataManager()
self.DataTable.setDataManager(self.dataManager)
self.MPLWidget.setDataManager(self.dataManager)
# self.VarGuesses.setDataManager(self.dataManager);
self.dataManager.attach(self.dataUpdated)
# sys.stderr = Stream(newText=self.onUpdateText)
sys.stdout = Stream(newText=self.onUpdateText)
def testOpenPDZFile(self):
pdzmanager = pdzutils.PDZManager.open(self.pdzfile)
manager = md.DataManager()
manager.open(name='pdzmanager',
file=self.pdzfile,
open=pdzutils.PDZManager.open)
self.assertTrue('pdzmanager' in manager)
self.assertEquals(pdzmanager, manager.pdzmanager)
def save_to_manager(self, path):
"""Save the internal state of this object to external storage."""
dm = datamanager.DataManager(path)
meta = {
'conf': self.conffile,
}
data = {
'roads': self.roads.lines,
'translation': self.translation,
'scaling': self.scaling
}
dm.set(metadata=meta, data=data)
def save_to_manager(self, path, no_save=False):
"""Saves the object to external storage."""
dm = datamanager.DataManager(path)
meta = {'discrete_map_type': 'DiscreteMap', 'name': self.name}
data = {
'img': self.img,
'roads': self.roads.lines,
'labeller': self.labeller.to_dict()
}
if no_save:
return dm, meta, data
else:
dm.set(metadata=meta, data=data)
def testOpenCatalog(self):
manager = md.DataManager()
manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile)
self.assertTrue('cat1' in manager)
self.assertTrue(hasattr(manager, 'cat1'))
self.assertTrue((manager.cat1['SeqNr'] == self.cat1['SeqNr']).all())
manager.open('cat2', self.cat2file, ldac.openObjectFile, 'STDTAB')
self.assertTrue('cat2' in manager)
self.assertTrue((manager.cat2['SeqNr'] == self.cat2['SeqNr']).all())
def testOpenHistory(self):
filestat = os.stat(self.cat2file)
manager = md.DataManager()
preTime = datetime.datetime.now()
manager.open('cat2', self.cat2file, ldac.openObjectFile, 'STDTAB')
postTime = datetime.datetime.now()
history = manager.history[-1]
self.assertTrue(preTime <= history.time <= postTime)
self.assertEquals(history.action, 'OPEN')
self.assertEquals(history.name, 'cat2')
self.assertEquals(history.file, self.cat2file)
self.assertEquals(history.filestat, filestat)
self.assertEquals(history.method, ldac.openObjectFile)
self.assertEquals(history.methodargs, ('STDTAB', ))
self.assertEquals(history.methodkeywords, {})
printedhistory = str(history)
print printedhistory
self.assertTrue(
time.asctime(history.time.timetuple()) in printedhistory)
self.assertTrue('cat2' in printedhistory)
self.assertTrue(os.path.abspath(self.cat2file) in printedhistory)
userid = pwd.getpwuid(filestat.st_uid)[0]
self.assertTrue(userid in printedhistory)
modtime = time.asctime(time.localtime(filestat.st_mtime))
self.assertTrue(modtime in printedhistory)
methodname = ldac.openObjectFile.__name__
self.assertTrue(methodname in printedhistory)
self.assertTrue(str(ldac.openObjectFile) not in printedhistory)
self.assertTrue(str(history.methodargs) in printedhistory)
self.assertTrue(str(history.methodkeywords) in printedhistory)
sourcefile = inspect.getfile(ldac.openObjectFile)
self.assertTrue(sourcefile in printedhistory)
sourcefile_moddate = time.asctime(time.localtime(filestat.st_mtime))
self.assertTrue(sourcefile_moddate in printedhistory)
def testReplace(self):
manager = md.DataManager()
manager.thing0 = np.zeros(15)
manager.replace('thing0', None)
self.assertTrue(manager.thing0 is None)
history = manager.history[-1]
self.assertEquals(history.name, 'thing0')
self.assertEquals(history.action, 'REPLACE')
def __init__(self, displayWidth, displayHeight, dataModule=None):
self.mode = None
self.displayWidth = displayWidth
self.displayHeight = displayHeight
pyui.desktop.getRenderer().setBackMethod(self.draw)
self.events = []
self.paused = 0
# initialize data
self.dm = datamanager.DataManager()
if dataModule:
dataModule.initialize(self.dm)
# create the world
self.world = world.World(displayWidth, displayHeight)
def testFunctionMultiStore(self):
manager = md.DataManager()
def makeThings(theamount):
return np.zeros(theamount), np.arange(theamount)
manager.store('thing1 thing2'.split(), makeThings, 5)
history = manager.history[-1]
printedhistory = str(history)
print printedhistory
self.assertTrue('thing1' in printedhistory)
self.assertTrue('thing2' in printedhistory)
self.assertTrue(str('thing1 thing2'.split()) not in printedhistory)
def testFilterWithArgs(self):
manager = md.DataManager()
manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile)
def filterFunction(seqnr, mod):
return seqnr % mod == 0
expected = manager.cat1.filter(filterFunction(manager.cat1['SeqNr'],
3))
manager.update(filterFunction, {'cat1': 'filter'},
manager.cat1['SeqNr'],
mod=3)
self.assertTrue((expected['SeqNr'] == manager.cat1['SeqNr']).all())
def checkin(name, project=None):
"""Check in a new shot or asset to the system."""
if not project:
project = utils.get_project()
print(project)
dm = datamanager.DataManager(project)
# Check to see if the asset or shot exists and create it if not
workon = dm.get_shot_or_asset(name)
node_settings = dm.settings.MAYA_NODE_PARSER
filelist = {}
for ntype, attr in node_settings.items():
for node in cmds.ls(et=ntype):
filelist[node] = cmds.getAttr("{0}.{1}".format(node, attr))
print(filelist)
def testStore(self):
manager = md.DataManager()
thing0 = np.zeros(5)
manager.store('thing0', thing0)
thing1 = np.zeros(15) + 1
thing2 = np.zeros(25) + 2
manager.store('thing1 thing2'.split(), (thing1, thing2))
self.assertTrue((manager.thing1 == thing1).all())
self.assertTrue((manager.thing2 == thing2).all())
self.assertRaises(md.ManagedDataException, manager.store, 'thing0',
thing1)
def save_to_manager(self, path, _no_save=False):
"""Saves the object to external storage."""
dm = datamanager.DataManager(path)
meta = {
'dmap_origin_path': path+'.dmap_origin',
'dmaps_paths': [
path+'.dmap{}'.format(i) for i in xrange(len(self.dmaps))
],
'grids_paths': [
[
path+'.grid{}_{}'.format(i, j)
for j in xrange(len(self.grids[i]))
] for i in xrange(len(self.grids))
],
'lrpd_config': self.lrpd_config
}
data = {
'scales': self.scales,
'smin_idx': self.smin_idx,
'smax_idx': self.smax_idx,
'valid_radii': self.valid_radii,
'radii': self.radii,
'rmin_idx': self.rmin_idx,
'rmax_idx': self.rmax_idx,
'descs': self.descs,
'descs_revidx': self.descs_revidx,
'angles': self.angles
}
# Save the original dmap
self.dmap_origin.save_to_manager(meta['dmap_origin_path'])
# Save the scaled dmaps
for i, dmap_path in enumerate(meta['dmaps_paths']):
self.dmaps[i].save_to_manager(dmap_path)
# Save the grids
for i in xrange(len(self.grids)):
for j in xrange(len(self.grids[i])):
self.grids[i][j].save_to_manager(
meta['grids_paths'][i][j]
)
if _no_save:
return dm, meta, data
else:
dm.set(metadata=meta, data=data)
def testInitHistory(self):
preTime = datetime.datetime.now()
manager = md.DataManager()
postTime = datetime.datetime.now()
historyitem = manager.history[-1]
self.assertTrue(preTime <= historyitem.time <= postTime)
self.assertEquals(historyitem.action, 'INIT')
printedhistory = str(historyitem)
print printedhistory
self.assertTrue('INIT' in printedhistory)
self.assertTrue(
time.asctime(historyitem.time.timetuple()) in printedhistory)
self.assertTrue(getpass.getuser() in printedhistory)
def _save_image(container_path, path_extension):
files, args = flask.request.files, flask.request.form
if 'token' not in args or 'compress_quality' not in args or 'image' not in files:
return random.choice(config.CURSES)
if args['token'] not in config_secure.ACCEPTED_TOKENS:
return random.choice(config.CURSES)
if files['image'].filename.split(
'.')[-1].lower() not in config.ACCEPTED_FORMATS:
return 'Wrong image format.'
data_manager = datamanager.DataManager(files['image'], container_path)
data_manager.save()
if files['image'].filename.split('.')[1] in config.COMPRESSIBLE_FORMATS:
imageprocess.compress_image(data_manager.path(),
int(args['compress_quality']))
return 'http://{}:{}/{}/{}'.format(config_secure.EXTERNAL_IP,
config_secure.DEPLOY_PORT,
path_extension,
data_manager.file_name())
def testStoreHistory(self):
manager = md.DataManager()
manager.thing0 = np.zeros(5)
historyitem = manager.history[-1]
self.assertEquals(historyitem.name, 'thing0')
self.assertEquals(historyitem.action, 'STORE')
printedhistory = historyitem.log()
print printedhistory
self.assertTrue(
time.asctime(historyitem.time.timetuple()) in printedhistory)
self.assertTrue(historyitem.name in printedhistory)
self.assertTrue('STORE' in printedhistory)
def testMultiStoreHistory(self):
manager = md.DataManager()
manager.store('thing1 thing2'.split(), (np.zeros(5), np.ones(5)))
historyitem = manager.history[-1]
self.assertEquals(historyitem.name, ['thing1', 'thing2'])
self.assertEquals(historyitem.action, 'STORE')
printedhistory = historyitem.log()
self.assertTrue(
time.asctime(historyitem.time.timetuple()) in printedhistory)
self.assertTrue('thing1' in printedhistory)
self.assertTrue('thing2' in printedhistory)
self.assertTrue(str('thing1 thing2'.split()) not in printedhistory)
self.assertTrue('STORE' in printedhistory)
def load_from_manager(cls, path, _return_dm=False):
"""Loads an object from external storage."""
dm = datamanager.DataManager(path)
meta, data = dm.get()
inst = RMSMap(None, None, None, _skip_init=True)
# Restore original dmap
inst.dmap_origin = maps.discretemap.DiscreteMap.load_from_manager(
meta['dmap_origin_path']
)
# Restore scaled dmaps
inst.dmaps = []
for dmap_path in meta['dmaps_paths']:
inst.dmaps.append(maps.discretemap.DiscreteMap.load_from_manager(
dmap_path
))
# Restore grids
inst.grids = []
for i, grids in enumerate(meta['grids_paths']):
inst.grids.append([])
for j, grid_path in enumerate(grids):
inst.grids[-1].append(util.grid.Grid.load_from_manager(
grid_path
))
# Restore lrpd_config
inst.lrpd_config = meta['lrpd_config']
# Restore all data
inst.scales = data['scales']
inst.smin_idx = data['smin_idx']
inst.smax_idx = data['smax_idx']
inst.valid_radii = data['valid_radii']
inst.radii = data['radii']
inst.rmin_idx = data['rmin_idx']
inst.rmax_idx = data['rmax_idx']
inst.descs = data['descs']
inst.descs_revidx = data['descs_revidx']
inst.angles = data['angles']
if _return_dm:
return dm, inst, meta, data
else:
return inst
def testFunctionStoreHistory(self):
manager = md.DataManager()
def makeThing(theamount):
return np.arange(theamount)
manager.store('thing1', makeThing, 5)
history = manager.history[-1]
self.assertEquals(history.name, 'thing1')
self.assertEquals(history.action, 'STORE')
self.assertEquals(history.method, makeThing)
self.assertEquals(history.methodargs, (5, ))
self.assertEquals(history.methodkeywords, {})
printedhistory = str(history)
print printedhistory
self.assertTrue('thing1' in printedhistory)
self.assertTrue('STORE' in printedhistory)
self.assertTrue(
time.asctime(history.time.timetuple()) in printedhistory)
methodname = makeThing.__name__
self.assertTrue(methodname in printedhistory)
self.assertTrue(str(makeThing) not in printedhistory)
sourcefile = inspect.getfile(makeThing)
self.assertTrue(sourcefile in printedhistory)
sourcefile_moddate = time.asctime(
time.localtime(os.stat(sourcefile).st_mtime))
self.assertTrue(sourcefile_moddate in printedhistory)
sourcecode = inspect.getsource(makeThing)
self.assertTrue(sourcecode in printedhistory)
self.assertTrue('args = %s' %
str(history.methodargs) in printedhistory)
self.assertTrue('keywords = %s' % str(history.methodkeywords))
def testComments(self):
manager = md.DataManager()
options = {'arg1': 5, 'arg2': 'agb'}
comment = 'Maxlike called with %s' % str(options)
manager.history.comment(comment)
history = manager.history[-1]
self.assertEquals(history.action, 'COMMENT')
self.assertEquals(history.comment, comment)
printedhistory = history.log()
print printedhistory
self.assertTrue(
time.asctime(history.time.timetuple()) in printedhistory)
self.assertTrue('COMMENT' in printedhistory)
self.assertTrue(comment in printedhistory)
def testStoreFuncResult(self):
manager = md.DataManager()
def makeThing(theamount):
return np.arange(theamount)
manager.store('thing1', makeThing, 5)
expected = makeThing(5)
self.assertTrue((manager.thing1 == expected).all())
def makeTwoThings(theamount):
return np.zeros(theamount), np.arange(theamount)
manager.store('thing2 thing3'.split(), makeTwoThings, theamount=5)
expected2, expected3 = makeTwoThings(5)
self.assertTrue((manager.thing2 == expected2).all())
self.assertTrue((manager.thing3 == expected3).all())
def add_result(self, params, results, metadata={}):
"""Add some results to storage.
Args:
params: the parameters used
results: the results obtained
metadata: a dictionary containing metadata about the results.
Existing metadata will be merged with what is provided (but only
within limits, ie key collisions will be overwritten with the
provided values)
"""
dm = datamanager.DataManager(self.path)
meta, data = {}, {'data': []}
if dm.exists():
meta, data = dm.get()
# Add/overwrite the existing metadata
for k, v in metadata.iteritems():
meta[k] = v
data['data'].append((params, results))
dm.set(metadata=meta, data=data)
def __init__(self):
self.DM = dm.DataManager()
# transaction_index : transaction
self.transaction_list = {}
# transaction_index(a) : [transaction_index(b)] transaction a waits b1,b2, ..., bi
self.transaction_wait_table = {}
# data(x) : [transaction_index(a)] transaction a1, a2,..., ai are waiting for locking data x
self.data_wait_table = {}
# transaction_index(b) : [transaction_index(ai)] (sequential list) b blocks a1, a2, ..., ai
self.block_table = {}
# site fail time
self.fail_history = {}
# final result for each transaction (commit/abort)
self.final_result = {}
# final committed value for each data
self.commit_summary = {}
def get_results(self, pf={}, rf={}):
"""Returns results, filtered as specified.
Args:
pf: a dictionary, where each key is of the form 'some.param.path',
and a value v. The available data will be filtered on the values
of params['some']['param']['path']: only results where this is
equal to v are kept.
All filter items must match for a (params, result) item to be kept.
A value of None means the parameter must NOT be set, or equal to
None.
rf: an iterable of paths of the the same form as pf, which filters
what will be included in the returned list of results
Returns:
A (meta, data) couple, where data is a [(params, result)] list,
filtered as specified.
"""
dm = datamanager.DataManager(self.path)
if not dm.exists():
return ({}, [])
meta, data = dm.get()
filtered_items = []
for params, result in data['data']:
add = True
for p_path, p_value in pf.iteritems():
if not self._is_path_equal(params, p_path, p_value):
add = False
break
if not add:
continue
filtered_items.append((params, self._filter_result(result, rf)))
return meta, filtered_items
