def testFilter(self): manager = md.DataManager() manager.thing0 = np.zeros(15) manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile) manager.open(name='pdzmanager', file=self.pdzfile, open=pdzutils.PDZManager.open) manager.store('pdzrange pdz'.split(), manager.pdzmanager.associatePDZ(manager.cat1['SeqNr'])) def filterFunction(manager): return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0] theFilter = filterFunction(manager) expectedSeq = manager.cat1.filter(theFilter)['SeqNr'] manager.update(filterFunction, { 'cat1': 'filter', 'pdz': '__getitem__' }) self.assertEquals(len(manager.thing0), 15) self.assertEquals(len(manager.pdzrange), 25) self.assertTrue((manager.cat1['SeqNr'] == expectedSeq).all()) self.assertEquals(manager.pdz.shape, (len(expectedSeq), 25))
def testLogStatement(self): manager = md.DataManager() manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile) manager.open(name='pdzmanager', file=self.pdzfile, open=pdzutils.PDZManager.open) manager.store('pdzrange pdz'.split(), manager.pdzmanager.associatePDZ(manager.cat1['SeqNr'])) def filterFunction(manager): return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0] manager.update(filterFunction, { 'cat1': 'filter', 'pdz': '__getitem__' }) self.assertEquals(len(manager.history), 5) log = manager.history.log() print log for historyitem in manager.history: self.assertTrue(historyitem.log() in log)
def __init__(self, data, ip, port, clientserver): self.data = data self._ip = ip self._port = port self._server = clientserver self.datamgr = datamanager.DataManager("localhost", "liwaa", "p@ss0rd", "testDatabase")
def testFilterHistory(self): manager = md.DataManager() manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile) manager.open(name='pdzmanager', file=self.pdzfile, open=pdzutils.PDZManager.open) manager.store('pdzrange pdz'.split(), manager.pdzmanager.associatePDZ(manager.cat1['SeqNr'])) def filterFunction(manager): return np.arange(len(manager.cat1))[manager.cat1['SeqNr'] % 2 == 0] preTime = datetime.datetime.now() manager.update(filterFunction, { 'cat1': 'filter', 'pdz': '__getitem__' }) postTime = datetime.datetime.now() history = manager.history[-1] self.assertTrue(preTime <= history.time <= postTime) self.assertEquals(history.action, 'UPDATE') self.assertEquals(history.name, ['cat1', 'pdz']) self.assertEquals(history.method, filterFunction) self.assertEquals(history.methodargs, ()) self.assertEquals(history.methodkeywords, {}) printedhistory = str(history) print printedhistory self.assertTrue( time.asctime(history.time.timetuple()) in printedhistory) self.assertTrue('UPDATE' in printedhistory) self.assertTrue('cat1' in printedhistory) self.assertTrue('pdz' in printedhistory) methodname = filterFunction.__name__ self.assertTrue(methodname in printedhistory) self.assertTrue(str(filterFunction) not in printedhistory) sourcefile = inspect.getfile(filterFunction) self.assertTrue(sourcefile in printedhistory) sourcefile_moddate = time.asctime( time.localtime(os.stat(sourcefile).st_mtime)) self.assertTrue(sourcefile_moddate in printedhistory) sourcecode = inspect.getsource(filterFunction) self.assertTrue(sourcecode in printedhistory) self.assertTrue(str(history.methodargs) in printedhistory) self.assertTrue(str(history.methodkeywords) in printedhistory)
def testImmutibility(self): manager = md.DataManager() manager.sizecol = 'rh' self.assertRaises(md.ManagedDataException, setattr, manager, 'sizecol', 32) self.assertRaises(md.ManagedDataException, manager.__setitem__, 'sizecol', 42)
def load_from_manager(cls, path): """Create an instance of this class by loading its data from files.""" dm = datamanager.DataManager(path) meta, data = dm.get() inst = cls(meta['conf'], _skip_init=True) inst.roads = geom.Segments(data['roads']) inst.translation = data['translation'] inst.scaling = data['scaling'] return inst
def __init__(self): super(ApplicationWindow, self).__init__() QtWidgets.QMainWindow.__init__(self) Ui_MainWindow.__init__(self) self.setupUi(self) self.dataManager = dm.DataManager() self.DataTable.setDataManager(self.dataManager) self.MPLWidget.setDataManager(self.dataManager) # self.VarGuesses.setDataManager(self.dataManager); self.dataManager.attach(self.dataUpdated) # sys.stderr = Stream(newText=self.onUpdateText) sys.stdout = Stream(newText=self.onUpdateText)
def testOpenPDZFile(self): pdzmanager = pdzutils.PDZManager.open(self.pdzfile) manager = md.DataManager() manager.open(name='pdzmanager', file=self.pdzfile, open=pdzutils.PDZManager.open) self.assertTrue('pdzmanager' in manager) self.assertEquals(pdzmanager, manager.pdzmanager)
def save_to_manager(self, path): """Save the internal state of this object to external storage.""" dm = datamanager.DataManager(path) meta = { 'conf': self.conffile, } data = { 'roads': self.roads.lines, 'translation': self.translation, 'scaling': self.scaling } dm.set(metadata=meta, data=data)
def save_to_manager(self, path, no_save=False): """Saves the object to external storage.""" dm = datamanager.DataManager(path) meta = {'discrete_map_type': 'DiscreteMap', 'name': self.name} data = { 'img': self.img, 'roads': self.roads.lines, 'labeller': self.labeller.to_dict() } if no_save: return dm, meta, data else: dm.set(metadata=meta, data=data)
def testOpenCatalog(self): manager = md.DataManager() manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile) self.assertTrue('cat1' in manager) self.assertTrue(hasattr(manager, 'cat1')) self.assertTrue((manager.cat1['SeqNr'] == self.cat1['SeqNr']).all()) manager.open('cat2', self.cat2file, ldac.openObjectFile, 'STDTAB') self.assertTrue('cat2' in manager) self.assertTrue((manager.cat2['SeqNr'] == self.cat2['SeqNr']).all())
def testOpenHistory(self): filestat = os.stat(self.cat2file) manager = md.DataManager() preTime = datetime.datetime.now() manager.open('cat2', self.cat2file, ldac.openObjectFile, 'STDTAB') postTime = datetime.datetime.now() history = manager.history[-1] self.assertTrue(preTime <= history.time <= postTime) self.assertEquals(history.action, 'OPEN') self.assertEquals(history.name, 'cat2') self.assertEquals(history.file, self.cat2file) self.assertEquals(history.filestat, filestat) self.assertEquals(history.method, ldac.openObjectFile) self.assertEquals(history.methodargs, ('STDTAB', )) self.assertEquals(history.methodkeywords, {}) printedhistory = str(history) print printedhistory self.assertTrue( time.asctime(history.time.timetuple()) in printedhistory) self.assertTrue('cat2' in printedhistory) self.assertTrue(os.path.abspath(self.cat2file) in printedhistory) userid = pwd.getpwuid(filestat.st_uid)[0] self.assertTrue(userid in printedhistory) modtime = time.asctime(time.localtime(filestat.st_mtime)) self.assertTrue(modtime in printedhistory) methodname = ldac.openObjectFile.__name__ self.assertTrue(methodname in printedhistory) self.assertTrue(str(ldac.openObjectFile) not in printedhistory) self.assertTrue(str(history.methodargs) in printedhistory) self.assertTrue(str(history.methodkeywords) in printedhistory) sourcefile = inspect.getfile(ldac.openObjectFile) self.assertTrue(sourcefile in printedhistory) sourcefile_moddate = time.asctime(time.localtime(filestat.st_mtime)) self.assertTrue(sourcefile_moddate in printedhistory)
def testReplace(self): manager = md.DataManager() manager.thing0 = np.zeros(15) manager.replace('thing0', None) self.assertTrue(manager.thing0 is None) history = manager.history[-1] self.assertEquals(history.name, 'thing0') self.assertEquals(history.action, 'REPLACE')
def __init__(self, displayWidth, displayHeight, dataModule=None): self.mode = None self.displayWidth = displayWidth self.displayHeight = displayHeight pyui.desktop.getRenderer().setBackMethod(self.draw) self.events = [] self.paused = 0 # initialize data self.dm = datamanager.DataManager() if dataModule: dataModule.initialize(self.dm) # create the world self.world = world.World(displayWidth, displayHeight)
def testFunctionMultiStore(self): manager = md.DataManager() def makeThings(theamount): return np.zeros(theamount), np.arange(theamount) manager.store('thing1 thing2'.split(), makeThings, 5) history = manager.history[-1] printedhistory = str(history) print printedhistory self.assertTrue('thing1' in printedhistory) self.assertTrue('thing2' in printedhistory) self.assertTrue(str('thing1 thing2'.split()) not in printedhistory)
def testFilterWithArgs(self): manager = md.DataManager() manager.open(name='cat1', file=self.cat1file, open=ldac.openObjectFile) def filterFunction(seqnr, mod): return seqnr % mod == 0 expected = manager.cat1.filter(filterFunction(manager.cat1['SeqNr'], 3)) manager.update(filterFunction, {'cat1': 'filter'}, manager.cat1['SeqNr'], mod=3) self.assertTrue((expected['SeqNr'] == manager.cat1['SeqNr']).all())
def checkin(name, project=None): """Check in a new shot or asset to the system.""" if not project: project = utils.get_project() print(project) dm = datamanager.DataManager(project) # Check to see if the asset or shot exists and create it if not workon = dm.get_shot_or_asset(name) node_settings = dm.settings.MAYA_NODE_PARSER filelist = {} for ntype, attr in node_settings.items(): for node in cmds.ls(et=ntype): filelist[node] = cmds.getAttr("{0}.{1}".format(node, attr)) print(filelist)
def testStore(self): manager = md.DataManager() thing0 = np.zeros(5) manager.store('thing0', thing0) thing1 = np.zeros(15) + 1 thing2 = np.zeros(25) + 2 manager.store('thing1 thing2'.split(), (thing1, thing2)) self.assertTrue((manager.thing1 == thing1).all()) self.assertTrue((manager.thing2 == thing2).all()) self.assertRaises(md.ManagedDataException, manager.store, 'thing0', thing1)
def save_to_manager(self, path, _no_save=False): """Saves the object to external storage.""" dm = datamanager.DataManager(path) meta = { 'dmap_origin_path': path+'.dmap_origin', 'dmaps_paths': [ path+'.dmap{}'.format(i) for i in xrange(len(self.dmaps)) ], 'grids_paths': [ [ path+'.grid{}_{}'.format(i, j) for j in xrange(len(self.grids[i])) ] for i in xrange(len(self.grids)) ], 'lrpd_config': self.lrpd_config } data = { 'scales': self.scales, 'smin_idx': self.smin_idx, 'smax_idx': self.smax_idx, 'valid_radii': self.valid_radii, 'radii': self.radii, 'rmin_idx': self.rmin_idx, 'rmax_idx': self.rmax_idx, 'descs': self.descs, 'descs_revidx': self.descs_revidx, 'angles': self.angles } # Save the original dmap self.dmap_origin.save_to_manager(meta['dmap_origin_path']) # Save the scaled dmaps for i, dmap_path in enumerate(meta['dmaps_paths']): self.dmaps[i].save_to_manager(dmap_path) # Save the grids for i in xrange(len(self.grids)): for j in xrange(len(self.grids[i])): self.grids[i][j].save_to_manager( meta['grids_paths'][i][j] ) if _no_save: return dm, meta, data else: dm.set(metadata=meta, data=data)
def testInitHistory(self): preTime = datetime.datetime.now() manager = md.DataManager() postTime = datetime.datetime.now() historyitem = manager.history[-1] self.assertTrue(preTime <= historyitem.time <= postTime) self.assertEquals(historyitem.action, 'INIT') printedhistory = str(historyitem) print printedhistory self.assertTrue('INIT' in printedhistory) self.assertTrue( time.asctime(historyitem.time.timetuple()) in printedhistory) self.assertTrue(getpass.getuser() in printedhistory)
def _save_image(container_path, path_extension): files, args = flask.request.files, flask.request.form if 'token' not in args or 'compress_quality' not in args or 'image' not in files: return random.choice(config.CURSES) if args['token'] not in config_secure.ACCEPTED_TOKENS: return random.choice(config.CURSES) if files['image'].filename.split( '.')[-1].lower() not in config.ACCEPTED_FORMATS: return 'Wrong image format.' data_manager = datamanager.DataManager(files['image'], container_path) data_manager.save() if files['image'].filename.split('.')[1] in config.COMPRESSIBLE_FORMATS: imageprocess.compress_image(data_manager.path(), int(args['compress_quality'])) return 'http://{}:{}/{}/{}'.format(config_secure.EXTERNAL_IP, config_secure.DEPLOY_PORT, path_extension, data_manager.file_name())
def testStoreHistory(self): manager = md.DataManager() manager.thing0 = np.zeros(5) historyitem = manager.history[-1] self.assertEquals(historyitem.name, 'thing0') self.assertEquals(historyitem.action, 'STORE') printedhistory = historyitem.log() print printedhistory self.assertTrue( time.asctime(historyitem.time.timetuple()) in printedhistory) self.assertTrue(historyitem.name in printedhistory) self.assertTrue('STORE' in printedhistory)
def testMultiStoreHistory(self): manager = md.DataManager() manager.store('thing1 thing2'.split(), (np.zeros(5), np.ones(5))) historyitem = manager.history[-1] self.assertEquals(historyitem.name, ['thing1', 'thing2']) self.assertEquals(historyitem.action, 'STORE') printedhistory = historyitem.log() self.assertTrue( time.asctime(historyitem.time.timetuple()) in printedhistory) self.assertTrue('thing1' in printedhistory) self.assertTrue('thing2' in printedhistory) self.assertTrue(str('thing1 thing2'.split()) not in printedhistory) self.assertTrue('STORE' in printedhistory)
def load_from_manager(cls, path, _return_dm=False): """Loads an object from external storage.""" dm = datamanager.DataManager(path) meta, data = dm.get() inst = RMSMap(None, None, None, _skip_init=True) # Restore original dmap inst.dmap_origin = maps.discretemap.DiscreteMap.load_from_manager( meta['dmap_origin_path'] ) # Restore scaled dmaps inst.dmaps = [] for dmap_path in meta['dmaps_paths']: inst.dmaps.append(maps.discretemap.DiscreteMap.load_from_manager( dmap_path )) # Restore grids inst.grids = [] for i, grids in enumerate(meta['grids_paths']): inst.grids.append([]) for j, grid_path in enumerate(grids): inst.grids[-1].append(util.grid.Grid.load_from_manager( grid_path )) # Restore lrpd_config inst.lrpd_config = meta['lrpd_config'] # Restore all data inst.scales = data['scales'] inst.smin_idx = data['smin_idx'] inst.smax_idx = data['smax_idx'] inst.valid_radii = data['valid_radii'] inst.radii = data['radii'] inst.rmin_idx = data['rmin_idx'] inst.rmax_idx = data['rmax_idx'] inst.descs = data['descs'] inst.descs_revidx = data['descs_revidx'] inst.angles = data['angles'] if _return_dm: return dm, inst, meta, data else: return inst
def testFunctionStoreHistory(self): manager = md.DataManager() def makeThing(theamount): return np.arange(theamount) manager.store('thing1', makeThing, 5) history = manager.history[-1] self.assertEquals(history.name, 'thing1') self.assertEquals(history.action, 'STORE') self.assertEquals(history.method, makeThing) self.assertEquals(history.methodargs, (5, )) self.assertEquals(history.methodkeywords, {}) printedhistory = str(history) print printedhistory self.assertTrue('thing1' in printedhistory) self.assertTrue('STORE' in printedhistory) self.assertTrue( time.asctime(history.time.timetuple()) in printedhistory) methodname = makeThing.__name__ self.assertTrue(methodname in printedhistory) self.assertTrue(str(makeThing) not in printedhistory) sourcefile = inspect.getfile(makeThing) self.assertTrue(sourcefile in printedhistory) sourcefile_moddate = time.asctime( time.localtime(os.stat(sourcefile).st_mtime)) self.assertTrue(sourcefile_moddate in printedhistory) sourcecode = inspect.getsource(makeThing) self.assertTrue(sourcecode in printedhistory) self.assertTrue('args = %s' % str(history.methodargs) in printedhistory) self.assertTrue('keywords = %s' % str(history.methodkeywords))
def testComments(self): manager = md.DataManager() options = {'arg1': 5, 'arg2': 'agb'} comment = 'Maxlike called with %s' % str(options) manager.history.comment(comment) history = manager.history[-1] self.assertEquals(history.action, 'COMMENT') self.assertEquals(history.comment, comment) printedhistory = history.log() print printedhistory self.assertTrue( time.asctime(history.time.timetuple()) in printedhistory) self.assertTrue('COMMENT' in printedhistory) self.assertTrue(comment in printedhistory)
def testStoreFuncResult(self): manager = md.DataManager() def makeThing(theamount): return np.arange(theamount) manager.store('thing1', makeThing, 5) expected = makeThing(5) self.assertTrue((manager.thing1 == expected).all()) def makeTwoThings(theamount): return np.zeros(theamount), np.arange(theamount) manager.store('thing2 thing3'.split(), makeTwoThings, theamount=5) expected2, expected3 = makeTwoThings(5) self.assertTrue((manager.thing2 == expected2).all()) self.assertTrue((manager.thing3 == expected3).all())
def add_result(self, params, results, metadata={}): """Add some results to storage. Args: params: the parameters used results: the results obtained metadata: a dictionary containing metadata about the results. Existing metadata will be merged with what is provided (but only within limits, ie key collisions will be overwritten with the provided values) """ dm = datamanager.DataManager(self.path) meta, data = {}, {'data': []} if dm.exists(): meta, data = dm.get() # Add/overwrite the existing metadata for k, v in metadata.iteritems(): meta[k] = v data['data'].append((params, results)) dm.set(metadata=meta, data=data)
def __init__(self): self.DM = dm.DataManager() # transaction_index : transaction self.transaction_list = {} # transaction_index(a) : [transaction_index(b)] transaction a waits b1,b2, ..., bi self.transaction_wait_table = {} # data(x) : [transaction_index(a)] transaction a1, a2,..., ai are waiting for locking data x self.data_wait_table = {} # transaction_index(b) : [transaction_index(ai)] (sequential list) b blocks a1, a2, ..., ai self.block_table = {} # site fail time self.fail_history = {} # final result for each transaction (commit/abort) self.final_result = {} # final committed value for each data self.commit_summary = {}
def get_results(self, pf={}, rf={}): """Returns results, filtered as specified. Args: pf: a dictionary, where each key is of the form 'some.param.path', and a value v. The available data will be filtered on the values of params['some']['param']['path']: only results where this is equal to v are kept. All filter items must match for a (params, result) item to be kept. A value of None means the parameter must NOT be set, or equal to None. rf: an iterable of paths of the the same form as pf, which filters what will be included in the returned list of results Returns: A (meta, data) couple, where data is a [(params, result)] list, filtered as specified. """ dm = datamanager.DataManager(self.path) if not dm.exists(): return ({}, []) meta, data = dm.get() filtered_items = [] for params, result in data['data']: add = True for p_path, p_value in pf.iteritems(): if not self._is_path_equal(params, p_path, p_value): add = False break if not add: continue filtered_items.append((params, self._filter_result(result, rf))) return meta, filtered_items