def getRelatedObjects(self, relation, obj, direction=em.RELATION_CHILDS): """ Get all objects related to obj by a give relation. Params: relation: the relation name to search for. obj: object from which the relation will be search, actually not only this, but all other objects connected to this one by the RELATION_TRANSFORM. direction: this say if search for childs or parents in the relation. """ graph = self.getTransformGraph() relations = self.mapper.getRelationsByName(relation) connection = self._getConnectedObjects(obj, graph) objects = [] objectsDict = {} for rel in relations: pObj = self.getObject(rel['object_parent_id']) pExt = rel['object_parent_extended'] pp = pwobj.Pointer(pObj, extended=pExt) if pp.getUniqueId() in connection: cObj = self.getObject(rel['object_child_id']) cExt = rel['object_child_extended'] cp = pwobj.Pointer(cObj, extended=cExt) if cp.hasValue() and cp.getUniqueId() not in objectsDict: objects.append(cp) objectsDict[cp.getUniqueId()] = True return objects
def _getRelationGraph(self, relation=em.RELATION_SOURCE, refresh=False): """ Retrieve objects produced as outputs and make a graph taking into account the SOURCE relation. """ relations = self.mapper.getRelationsByName(relation) g = pwutils.graph.Graph(rootName='PROJECT') root = g.getRoot() root.pointer = None runs = self.getRuns(refresh=refresh) for r in runs: for paramName, attr in r.iterOutputAttributes(em.EMObject): p = pwobj.Pointer(r, extended=paramName) node = g.createNode(p.getUniqueId(), attr.getNameId()) node.pointer = p # The following alias if for backward compatibility p2 = pwobj.Pointer(attr) g.aliasNode(node, p2.getUniqueId()) for rel in relations: pObj = self.getObject(rel['object_parent_id']) pExt = rel['object_parent_extended'] pp = pwobj.Pointer(pObj, extended=pExt) pid = pp.getUniqueId() parent = g.getNode(pid) while not parent and pp.hasExtended(): pp.removeExtended() parent = g.getNode(pp.getUniqueId()) if not parent: print "project._getRelationGraph: ERROR, parent Node is None: ", pid else: cObj = self.getObject(rel['object_child_id']) if cObj: cExt = rel['object_child_extended'] cp = pwobj.Pointer(cObj, extended=cExt) child = g.getNode(cp.getUniqueId()) if not child: print "project._getRelationGraph: ERROR, child Node is None: ", cp.getUniqueId( ) print " parent: ", pid else: parent.addChild(child) else: print "project._getRelationGraph: ERROR, child Obj is None, id: ", rel[ 'object_child_id'] print " parent: ", pid for n in g.getNodes(): if n.isRoot() and not n is root: root.addChild(n) return g
def test_StorePointers(self): """ Check that pointers are correctly stored. """ fn = self.getOutputPath("pointers.sqlite") print(">>> Using db: ", fn) mapper = pwmapper.SqliteMapper(fn) # Insert a Complex c = Complex.createComplex() # real = 1, imag = 1 mapper.insert(c) # Insert an pwobj.Integer p1 = pwobj.Pointer(c) p1.setExtended('real') mapper.store(c) mapper.store(p1) self.assertAlmostEqual(c.real.get(), p1.get().get()) p1.set(None) # Reset value and check that is stored properly self.assertIsNone(p1._extended.get()) mapper.store(p1) mapper.commit() mapper2 = pwmapper.SqliteMapper(fn, pw.Config.getDomain().getMapperDict()) p2 = mapper2.selectByClass('Pointer')[0] # Check the mapper was properly stored when # set to None and the _extended property cleaned self.assertIsNone(p2.get())
def registerCoords(self, coordsDir): """ This method is usually inherited by all Pickers and it is used from the Java picking GUI to register a new SetOfCoordinates when the user click on +Particles button. """ from pwem import Domain suffix = self.__getOutputSuffix() outputName = self.OUTPUT_PREFIX + suffix readSetOfCoordinates = Domain.importFromPlugin('xmipp3.convert', 'readSetOfCoordinates') inputset = self.getInputMicrographs() # micrographs are the input set if protocol is not finished outputset = self._createSetOfCoordinates(inputset, suffix=suffix) readSetOfCoordinates(coordsDir, outputset.getMicrographs(), outputset) summary = self.getSummary(outputset) outputset.setObjComment(summary) outputs = {outputName: outputset} self._defineOutputs(**outputs) # Using a pointer to define the relations is more robust to scheduling # and id changes between the protocol run.db and the main project # database. The pointer defined below points to the outputset object self._defineSourceRelation( self.getInputMicrographsPointer(), pwobj.Pointer(value=self, extended=outputName)) self._store()
def __init__(self, **kwargs): MockObject.__init__(self, **kwargs) self._micrographPointer = pwobj.Pointer(objDoStore=False) self._x = pwobj.Integer(kwargs.get('x', None)) self._y = pwobj.Integer(kwargs.get('y', None)) self._micId = pwobj.Integer() self._micName = pwobj.String()
def __init__(self, **kwargs): Volume.__init__(self, **kwargs) self._acquisition = None self._tsId = pwobj.String(kwargs.get('tsId', None)) self._tomoPointer = pwobj.Pointer(objDoStore=False) self._tomoId = pwobj.Integer() self._tomoName = pwobj.String()
def show(self, form): if form.protocol.inputPDB.hasValue(): pdb = form.protocol.inputPDB.get() print("pdb ", pdb._volume, str(pdb._volume)) if pdb._volume: print("Setting ", str(pdb.getVolume())) ptr = pwobj.Pointer() ptr.copy(form.protocol.inputPDB) ptr.setExtended(ptr.getExtended() + "._volume") # ptr.set(pdb.getVolume()) form.setVar('inputVol', ptr)
def test_basicObjectInProject(self): prot = self.newProtocol(ProtOutputTest, objLabel='to generate basic input') print("working dir: %s" % prot.getWorkingDir()) # Define a negative output for later tests prot._defineOutputs(negative=pwobj.Integer(-20)) self.launchProtocol(prot) # Default value is 10 so output is 20 self.assertOutput(prot) # Second protocol to test linking prot2 = self.newProtocol(ProtOutputTest, objLabel='to read basic input') # Set the pointer for the integer prot2.iBoxSize.setPointer(pwobj.Pointer(prot, extended="oBoxSize")) self.launchProtocol(prot2) self.assertOutput(prot2, value=40) # Test validation: only positive numbers are allowed prot3 = self.newProtocol(ProtOutputTest, objLabel='invalid input', iBoxSize=-10) # We expect this to fail with self.assertRaises(Exception): self.launchProtocol(prot3) # Test validation: pointer value is validated prot4 = self.newProtocol(ProtOutputTest, objLabel='invalid pointer input') # Now use negative pointer output prot4.iBoxSize.setPointer(pwobj.Pointer(prot, extended="negative")) # We expect this to fail with self.assertRaises(Exception): self.launchProtocol(prot4)
def testWithPointer(self): obj = pwobj.Integer(10) self.assertFalse(obj.hasPointer(), "Default instantiation off Integer has a pointer.") self.assertEqual(obj.get(), 10, "Integer.get(), without pointer fails.") pointee = pwobj.Object() setattr(pointee, "value", pwobj.Integer(20)) # Set a pointer (not a real case though, but enough here) obj.setPointer(pwobj.Pointer(pointee, extended='value')) self.assertEqual(obj.get(), 20, "Integer.get() fails with a pointer.")
def test_SqliteMapper(self): fn = self.getOutputPath("basic.sqlite") mapper = pwmapper.SqliteMapper(fn) # Insert a Float f = pwobj.Float(5.4) mapper.insert(f) # Insert an pwobj.Integer i = pwobj.Integer(1) mapper.insert(i) # Insert two pwobj.Boolean b = pwobj.Boolean(False) b2 = pwobj.Boolean(True) mapper.insert(b) mapper.insert(b2) # Test storing pointers p = pwobj.Pointer(b) mapper.insert(p) # Store csv list strList = ['1', '2', '3'] csv = pwobj.CsvList() csv += strList mapper.insert(csv) # Test normal List iList = pwobj.List() mapper.insert(iList) # Insert the list when empty i1 = pwobj.Integer(4) i2 = pwobj.Integer(3) iList.append(i1) iList.append(i2) mapper.update(iList) # now update with some items inside pList = pwobj.PointerList() p1 = pwobj.Pointer(b) # p1.set(b) p2 = pwobj.Pointer(b2) # p2.set(b2) pList.append(p1) pList.append(p2) mapper.store(pList) # Test to add relations relName = 'testRelation' creator = f mapper.insertRelation(relName, creator, i, b) mapper.insertRelation(relName, creator, i, b2) mapper.insertRelation(relName, creator, b, p) mapper.insertRelation(relName, creator, b2, p) # Save changes to file mapper.commit() self.assertEqual(1, mapper.db.getVersion()) mapper.close() # TODO: Maybe some mapper test for backward compatibility can be # include in scipion-em, where we already have defined datasets # and reference old sqlite files # Test using SqliteDb class db = pwmapper.SqliteDb() db._createConnection(fn, timeout=1000) tables = ['Objects', 'Relations'] self.assertEqual(tables, db.getTables()) # Test getting the version, for the gold file it should be 0 self.assertEqual(1, db.getVersion()) db.close() # Reading test mapper2 = pwmapper.SqliteMapper(fn, pw.Config.getDomain().getMapperDict()) print("Checking that Relations table is updated and version to 1") self.assertEqual(1, mapper2.db.getVersion()) # Check that the new column is properly added after updated to version 1 colNamesGold = [u'id', u'parent_id', u'name', u'classname', u'value', u'label', u'comment', u'object_parent_id', u'object_child_id', u'creation', u'object_parent_extended', u'object_child_extended'] colNames = [col[1] for col in mapper2.db.getTableColumns('Relations')] self.assertEqual(colNamesGold, colNames) l = mapper2.selectByClass('Integer')[0] self.assertEqual(l.get(), 1) f2 = mapper2.selectByClass('Float')[0] self.assertEqual(f, f2.get()) b = mapper2.selectByClass('Boolean')[0] self.assertTrue(not b.get()) p = mapper2.selectByClass('Pointer')[0] self.assertEqual(b.get(), p.get()) csv2 = mapper2.selectByClass('CsvList')[0] self.assertTrue(list.__eq__(csv2, strList)) # Iterate over all objects allObj = mapper2.selectAll() iterAllObj = mapper2.selectAll(iterate=True) for a1, a2 in zip(allObj, iterAllObj): # Note compare the scalar objects, which have a well-defined comparison if isinstance(a1, pwobj.Scalar): self.assertEqual(a1, a2) # Test select all batch approach allBatch = mapper2.selectAllBatch() # Test relations childs = mapper2.getRelationChilds(relName, i) parents = mapper2.getRelationParents(relName, p) # In this case both childs and parent should be the same for c, p in zip(childs, parents): self.assertEqual(c, p, "Childs of object i, should be the parents of object p") relations = mapper2.getRelationsByCreator(creator) for row in relations: print(dict(row))
def loadProtocols(self, filename=None, jsonStr=None): """ Load protocols generated in the same format as self.exportProtocols. Params: filename: the path of the file where to read the workflow. jsonStr: read the protocols from a string instead of file. Note: either filename or jsonStr should be not None. """ f = open(filename) protocolsList = json.load(f) emProtocols = em.getProtocols() newDict = OrderedDict() # First iteration: create all protocols and setup parameters for protDict in protocolsList: protClassName = protDict['object.className'] protId = protDict['object.id'] protClass = emProtocols.get(protClassName, None) if protClass is None: print "ERROR: protocol class name '%s' not found" % protClassName else: prot = self.newProtocol(protClass, objLabel=protDict.get('object.label', None), objComment=protDict.get( 'object.comment', None)) newDict[protId] = prot self.saveProtocol(prot) # Second iteration: update pointers values def _setPointer(pointer, value): # Properly setup the pointer value checking if the # id is already present in the dictionary parts = value.split('.') target = newDict.get(parts[0], None) pointer.set(target) if not pointer.pointsNone(): pointer.setExtendedParts(parts[1:]) for protDict in protocolsList: protId = protDict['object.id'] if protId in newDict: prot = newDict[protId] for paramName, attr in prot.iterDefinitionAttributes(): if paramName in protDict: # If the attribute is a pointer, we should look # if the id is already in the dictionary and # set the extended property if attr.isPointer(): _setPointer(attr, protDict[paramName]) # This case is similar to Pointer, but the values # is a list and we will setup a pointer for each value elif isinstance(attr, pwobj.PointerList): for value in protDict[paramName]: p = pwobj.Pointer() _setPointer(p, value) attr.append(p) # For "normal" parameters we just set the string value else: attr.set(protDict[paramName]) self.mapper.store(prot) f.close() self.mapper.commit() return newDict
def __init__(self, **kwargs): MockSet.__init__(self, **kwargs) self._micrographsPointer = pwobj.Pointer() self._boxSize = pwobj.Integer()
def __init__(self, **kwargs): MockSetOfImages.__init__(self, **kwargs) self._coordsPointer = pwobj.Pointer()
def test_Pointer(self): c = Complex.createComplex() p = pwobj.Pointer() p.set(c) p.setExtended('Name') c.Name = pwobj.String('Paquito') self.assertEqual(p.get(), 'Paquito') stackFn = "images.stk" mrcsFn = "images.mrcs" fn = self.getOutputPath('test_images.sqlite') imgSet = MockSetOfImages(filename=fn) imgSet.setSamplingRate(1.0) for i in range(10): img = MockImage() img.setLocation(i+1, stackFn) imgSet.append(img) imgSet.write() # Test that image number 7 is correctly retrieved # from the set img7 = imgSet[7] self.assertEqual(img7.getFileName(), stackFn) # Modify some properties of image 7 to test update img7.setFileName(mrcsFn) img7.setSamplingRate(2.0) imgSet.update(img7) # Write changes after the image 7 update imgSet.write() # Read again the set to be able to retrieve elements imgSet = MockSetOfImages(filename=fn) # Validate that image7 was properly updated img7 = imgSet[7] self.assertEqual(img7.getFileName(), mrcsFn) o = MockObject() o.pointer = pwobj.Pointer() o.pointer.set(imgSet) o.refC = o.pointer.get() attrNames = [k for k, a in o.getAttributes()] # Check that 'refC' should not appear in attributes # since it is only an "alias" to an existing pointed value self.assertNotIn('refC', attrNames) self.assertFalse(o.pointer.hasExtended(), 'o.pointer should not have extended at this point') o.pointer.setExtended(7) self.assertTrue(o.pointer.hasExtended()) self.assertTrue(o.pointer.hasExtended()) self.assertEqual(o.pointer.getExtended(), "7") # Check that the Item 7 of the set is properly # retrieved by the pointer after setting the extended to 7 self.assertEqual(imgSet[7], o.pointer.get()) # Test the keyword arguments of Pointer constructor # repeat above tests with new pointer ptr = pwobj.Pointer(value=imgSet, extended=7) self.assertTrue(ptr.hasExtended()) self.assertTrue(ptr.hasExtended()) self.assertEqual(ptr.getExtended(), "7") # Check that the Item 7 of the set is properly # retrieved by the pointer after setting the extended to 7 self.assertEqual(imgSet[7], ptr.get()) o2 = pwobj.OrderedObject() o2.outputImages = imgSet ptr2 = pwobj.Pointer() ptr2.set(o2) # Test nested extended attributes ptr2.setExtended('outputImages.7') self.assertEqual(imgSet[7], ptr2.get()) # Same as ptr2, but setting extended in constructor ptr3 = pwobj.Pointer(value=o2, extended='outputImages.7') self.assertEqual(imgSet[7], ptr3.get()) # Test copy between pointer objects ptr4 = pwobj.Pointer() ptr4.copy(ptr3) self.assertEqual(imgSet[7], ptr4.get()) self.assertEqual(ptr4.getExtended(), 'outputImages.7')