def getRelatedObjects(self, relation, obj, direction=em.RELATION_CHILDS):
""" Get all objects related to obj by a give relation.
Params:
relation: the relation name to search for.
obj: object from which the relation will be search,
actually not only this, but all other objects connected
to this one by the RELATION_TRANSFORM.
direction: this say if search for childs or parents in the relation.
"""
graph = self.getTransformGraph()
relations = self.mapper.getRelationsByName(relation)
connection = self._getConnectedObjects(obj, graph)
objects = []
objectsDict = {}
for rel in relations:
pObj = self.getObject(rel['object_parent_id'])
pExt = rel['object_parent_extended']
pp = pwobj.Pointer(pObj, extended=pExt)
if pp.getUniqueId() in connection:
cObj = self.getObject(rel['object_child_id'])
cExt = rel['object_child_extended']
cp = pwobj.Pointer(cObj, extended=cExt)
if cp.hasValue() and cp.getUniqueId() not in objectsDict:
objects.append(cp)
objectsDict[cp.getUniqueId()] = True
return objects
def _getRelationGraph(self, relation=em.RELATION_SOURCE, refresh=False):
""" Retrieve objects produced as outputs and
make a graph taking into account the SOURCE relation. """
relations = self.mapper.getRelationsByName(relation)
g = pwutils.graph.Graph(rootName='PROJECT')
root = g.getRoot()
root.pointer = None
runs = self.getRuns(refresh=refresh)
for r in runs:
for paramName, attr in r.iterOutputAttributes(em.EMObject):
p = pwobj.Pointer(r, extended=paramName)
node = g.createNode(p.getUniqueId(), attr.getNameId())
node.pointer = p
# The following alias if for backward compatibility
p2 = pwobj.Pointer(attr)
g.aliasNode(node, p2.getUniqueId())
for rel in relations:
pObj = self.getObject(rel['object_parent_id'])
pExt = rel['object_parent_extended']
pp = pwobj.Pointer(pObj, extended=pExt)
pid = pp.getUniqueId()
parent = g.getNode(pid)
while not parent and pp.hasExtended():
pp.removeExtended()
parent = g.getNode(pp.getUniqueId())
if not parent:
print "project._getRelationGraph: ERROR, parent Node is None: ", pid
else:
cObj = self.getObject(rel['object_child_id'])
if cObj:
cExt = rel['object_child_extended']
cp = pwobj.Pointer(cObj, extended=cExt)
child = g.getNode(cp.getUniqueId())
if not child:
print "project._getRelationGraph: ERROR, child Node is None: ", cp.getUniqueId(
)
print " parent: ", pid
else:
parent.addChild(child)
else:
print "project._getRelationGraph: ERROR, child Obj is None, id: ", rel[
'object_child_id']
print " parent: ", pid
for n in g.getNodes():
if n.isRoot() and not n is root:
root.addChild(n)
return g
def test_StorePointers(self):
""" Check that pointers are correctly stored. """
fn = self.getOutputPath("pointers.sqlite")
print(">>> Using db: ", fn)
mapper = pwmapper.SqliteMapper(fn)
# Insert a Complex
c = Complex.createComplex() # real = 1, imag = 1
mapper.insert(c)
# Insert an pwobj.Integer
p1 = pwobj.Pointer(c)
p1.setExtended('real')
mapper.store(c)
mapper.store(p1)
self.assertAlmostEqual(c.real.get(), p1.get().get())
p1.set(None) # Reset value and check that is stored properly
self.assertIsNone(p1._extended.get())
mapper.store(p1)
mapper.commit()
mapper2 = pwmapper.SqliteMapper(fn, pw.Config.getDomain().getMapperDict())
p2 = mapper2.selectByClass('Pointer')[0]
# Check the mapper was properly stored when
# set to None and the _extended property cleaned
self.assertIsNone(p2.get())
def registerCoords(self, coordsDir):
""" This method is usually inherited by all Pickers
and it is used from the Java picking GUI to register
a new SetOfCoordinates when the user click on +Particles button.
"""
from pwem import Domain
suffix = self.__getOutputSuffix()
outputName = self.OUTPUT_PREFIX + suffix
readSetOfCoordinates = Domain.importFromPlugin('xmipp3.convert',
'readSetOfCoordinates')
inputset = self.getInputMicrographs()
# micrographs are the input set if protocol is not finished
outputset = self._createSetOfCoordinates(inputset, suffix=suffix)
readSetOfCoordinates(coordsDir, outputset.getMicrographs(), outputset)
summary = self.getSummary(outputset)
outputset.setObjComment(summary)
outputs = {outputName: outputset}
self._defineOutputs(**outputs)
# Using a pointer to define the relations is more robust to scheduling
# and id changes between the protocol run.db and the main project
# database. The pointer defined below points to the outputset object
self._defineSourceRelation(
self.getInputMicrographsPointer(),
pwobj.Pointer(value=self, extended=outputName))
self._store()
def __init__(self, **kwargs):
MockObject.__init__(self, **kwargs)
self._micrographPointer = pwobj.Pointer(objDoStore=False)
self._x = pwobj.Integer(kwargs.get('x', None))
self._y = pwobj.Integer(kwargs.get('y', None))
self._micId = pwobj.Integer()
self._micName = pwobj.String()
def __init__(self, **kwargs):
Volume.__init__(self, **kwargs)
self._acquisition = None
self._tsId = pwobj.String(kwargs.get('tsId', None))
self._tomoPointer = pwobj.Pointer(objDoStore=False)
self._tomoId = pwobj.Integer()
self._tomoName = pwobj.String()
def show(self, form):
if form.protocol.inputPDB.hasValue():
pdb = form.protocol.inputPDB.get()
print("pdb ", pdb._volume, str(pdb._volume))
if pdb._volume:
print("Setting ", str(pdb.getVolume()))
ptr = pwobj.Pointer()
ptr.copy(form.protocol.inputPDB)
ptr.setExtended(ptr.getExtended() + "._volume")
# ptr.set(pdb.getVolume())
form.setVar('inputVol', ptr)
def test_basicObjectInProject(self):
prot = self.newProtocol(ProtOutputTest,
objLabel='to generate basic input')
print("working dir: %s" % prot.getWorkingDir())
# Define a negative output for later tests
prot._defineOutputs(negative=pwobj.Integer(-20))
self.launchProtocol(prot)
# Default value is 10 so output is 20
self.assertOutput(prot)
# Second protocol to test linking
prot2 = self.newProtocol(ProtOutputTest,
objLabel='to read basic input')
# Set the pointer for the integer
prot2.iBoxSize.setPointer(pwobj.Pointer(prot, extended="oBoxSize"))
self.launchProtocol(prot2)
self.assertOutput(prot2, value=40)
# Test validation: only positive numbers are allowed
prot3 = self.newProtocol(ProtOutputTest,
objLabel='invalid input',
iBoxSize=-10)
# We expect this to fail
with self.assertRaises(Exception):
self.launchProtocol(prot3)
# Test validation: pointer value is validated
prot4 = self.newProtocol(ProtOutputTest,
objLabel='invalid pointer input')
# Now use negative pointer output
prot4.iBoxSize.setPointer(pwobj.Pointer(prot, extended="negative"))
# We expect this to fail
with self.assertRaises(Exception):
self.launchProtocol(prot4)
def testWithPointer(self):
obj = pwobj.Integer(10)
self.assertFalse(obj.hasPointer(),
"Default instantiation off Integer has a pointer.")
self.assertEqual(obj.get(), 10, "Integer.get(), without pointer fails.")
pointee = pwobj.Object()
setattr(pointee, "value", pwobj.Integer(20))
# Set a pointer (not a real case though, but enough here)
obj.setPointer(pwobj.Pointer(pointee, extended='value'))
self.assertEqual(obj.get(), 20, "Integer.get() fails with a pointer.")
def test_SqliteMapper(self):
fn = self.getOutputPath("basic.sqlite")
mapper = pwmapper.SqliteMapper(fn)
# Insert a Float
f = pwobj.Float(5.4)
mapper.insert(f)
# Insert an pwobj.Integer
i = pwobj.Integer(1)
mapper.insert(i)
# Insert two pwobj.Boolean
b = pwobj.Boolean(False)
b2 = pwobj.Boolean(True)
mapper.insert(b)
mapper.insert(b2)
# Test storing pointers
p = pwobj.Pointer(b)
mapper.insert(p)
# Store csv list
strList = ['1', '2', '3']
csv = pwobj.CsvList()
csv += strList
mapper.insert(csv)
# Test normal List
iList = pwobj.List()
mapper.insert(iList) # Insert the list when empty
i1 = pwobj.Integer(4)
i2 = pwobj.Integer(3)
iList.append(i1)
iList.append(i2)
mapper.update(iList) # now update with some items inside
pList = pwobj.PointerList()
p1 = pwobj.Pointer(b)
# p1.set(b)
p2 = pwobj.Pointer(b2)
# p2.set(b2)
pList.append(p1)
pList.append(p2)
mapper.store(pList)
# Test to add relations
relName = 'testRelation'
creator = f
mapper.insertRelation(relName, creator, i, b)
mapper.insertRelation(relName, creator, i, b2)
mapper.insertRelation(relName, creator, b, p)
mapper.insertRelation(relName, creator, b2, p)
# Save changes to file
mapper.commit()
self.assertEqual(1, mapper.db.getVersion())
mapper.close()
# TODO: Maybe some mapper test for backward compatibility can be
# include in scipion-em, where we already have defined datasets
# and reference old sqlite files
# Test using SqliteDb class
db = pwmapper.SqliteDb()
db._createConnection(fn, timeout=1000)
tables = ['Objects', 'Relations']
self.assertEqual(tables, db.getTables())
# Test getting the version, for the gold file it should be 0
self.assertEqual(1, db.getVersion())
db.close()
# Reading test
mapper2 = pwmapper.SqliteMapper(fn, pw.Config.getDomain().getMapperDict())
print("Checking that Relations table is updated and version to 1")
self.assertEqual(1, mapper2.db.getVersion())
# Check that the new column is properly added after updated to version 1
colNamesGold = [u'id', u'parent_id', u'name', u'classname',
u'value', u'label', u'comment', u'object_parent_id',
u'object_child_id', u'creation',
u'object_parent_extended', u'object_child_extended']
colNames = [col[1] for col in mapper2.db.getTableColumns('Relations')]
self.assertEqual(colNamesGold, colNames)
l = mapper2.selectByClass('Integer')[0]
self.assertEqual(l.get(), 1)
f2 = mapper2.selectByClass('Float')[0]
self.assertEqual(f, f2.get())
b = mapper2.selectByClass('Boolean')[0]
self.assertTrue(not b.get())
p = mapper2.selectByClass('Pointer')[0]
self.assertEqual(b.get(), p.get())
csv2 = mapper2.selectByClass('CsvList')[0]
self.assertTrue(list.__eq__(csv2, strList))
# Iterate over all objects
allObj = mapper2.selectAll()
iterAllObj = mapper2.selectAll(iterate=True)
for a1, a2 in zip(allObj, iterAllObj):
# Note compare the scalar objects, which have a well-defined comparison
if isinstance(a1, pwobj.Scalar):
self.assertEqual(a1, a2)
# Test select all batch approach
allBatch = mapper2.selectAllBatch()
# Test relations
childs = mapper2.getRelationChilds(relName, i)
parents = mapper2.getRelationParents(relName, p)
# In this case both childs and parent should be the same
for c, p in zip(childs, parents):
self.assertEqual(c, p,
"Childs of object i, should be the parents of object p")
relations = mapper2.getRelationsByCreator(creator)
for row in relations:
print(dict(row))
def loadProtocols(self, filename=None, jsonStr=None):
""" Load protocols generated in the same format as self.exportProtocols.
Params:
filename: the path of the file where to read the workflow.
jsonStr: read the protocols from a string instead of file.
Note: either filename or jsonStr should be not None.
"""
f = open(filename)
protocolsList = json.load(f)
emProtocols = em.getProtocols()
newDict = OrderedDict()
# First iteration: create all protocols and setup parameters
for protDict in protocolsList:
protClassName = protDict['object.className']
protId = protDict['object.id']
protClass = emProtocols.get(protClassName, None)
if protClass is None:
print "ERROR: protocol class name '%s' not found" % protClassName
else:
prot = self.newProtocol(protClass,
objLabel=protDict.get('object.label',
None),
objComment=protDict.get(
'object.comment', None))
newDict[protId] = prot
self.saveProtocol(prot)
# Second iteration: update pointers values
def _setPointer(pointer, value):
# Properly setup the pointer value checking if the
# id is already present in the dictionary
parts = value.split('.')
target = newDict.get(parts[0], None)
pointer.set(target)
if not pointer.pointsNone():
pointer.setExtendedParts(parts[1:])
for protDict in protocolsList:
protId = protDict['object.id']
if protId in newDict:
prot = newDict[protId]
for paramName, attr in prot.iterDefinitionAttributes():
if paramName in protDict:
# If the attribute is a pointer, we should look
# if the id is already in the dictionary and
# set the extended property
if attr.isPointer():
_setPointer(attr, protDict[paramName])
# This case is similar to Pointer, but the values
# is a list and we will setup a pointer for each value
elif isinstance(attr, pwobj.PointerList):
for value in protDict[paramName]:
p = pwobj.Pointer()
_setPointer(p, value)
attr.append(p)
# For "normal" parameters we just set the string value
else:
attr.set(protDict[paramName])
self.mapper.store(prot)
f.close()
self.mapper.commit()
return newDict
def __init__(self, **kwargs):
MockSet.__init__(self, **kwargs)
self._micrographsPointer = pwobj.Pointer()
self._boxSize = pwobj.Integer()
def __init__(self, **kwargs):
MockSetOfImages.__init__(self, **kwargs)
self._coordsPointer = pwobj.Pointer()
def test_Pointer(self):
c = Complex.createComplex()
p = pwobj.Pointer()
p.set(c)
p.setExtended('Name')
c.Name = pwobj.String('Paquito')
self.assertEqual(p.get(), 'Paquito')
stackFn = "images.stk"
mrcsFn = "images.mrcs"
fn = self.getOutputPath('test_images.sqlite')
imgSet = MockSetOfImages(filename=fn)
imgSet.setSamplingRate(1.0)
for i in range(10):
img = MockImage()
img.setLocation(i+1, stackFn)
imgSet.append(img)
imgSet.write()
# Test that image number 7 is correctly retrieved
# from the set
img7 = imgSet[7]
self.assertEqual(img7.getFileName(), stackFn)
# Modify some properties of image 7 to test update
img7.setFileName(mrcsFn)
img7.setSamplingRate(2.0)
imgSet.update(img7)
# Write changes after the image 7 update
imgSet.write()
# Read again the set to be able to retrieve elements
imgSet = MockSetOfImages(filename=fn)
# Validate that image7 was properly updated
img7 = imgSet[7]
self.assertEqual(img7.getFileName(), mrcsFn)
o = MockObject()
o.pointer = pwobj.Pointer()
o.pointer.set(imgSet)
o.refC = o.pointer.get()
attrNames = [k for k, a in o.getAttributes()]
# Check that 'refC' should not appear in attributes
# since it is only an "alias" to an existing pointed value
self.assertNotIn('refC', attrNames)
self.assertFalse(o.pointer.hasExtended(),
'o.pointer should not have extended at this point')
o.pointer.setExtended(7)
self.assertTrue(o.pointer.hasExtended())
self.assertTrue(o.pointer.hasExtended())
self.assertEqual(o.pointer.getExtended(), "7")
# Check that the Item 7 of the set is properly
# retrieved by the pointer after setting the extended to 7
self.assertEqual(imgSet[7], o.pointer.get())
# Test the keyword arguments of Pointer constructor
# repeat above tests with new pointer
ptr = pwobj.Pointer(value=imgSet, extended=7)
self.assertTrue(ptr.hasExtended())
self.assertTrue(ptr.hasExtended())
self.assertEqual(ptr.getExtended(), "7")
# Check that the Item 7 of the set is properly
# retrieved by the pointer after setting the extended to 7
self.assertEqual(imgSet[7], ptr.get())
o2 = pwobj.OrderedObject()
o2.outputImages = imgSet
ptr2 = pwobj.Pointer()
ptr2.set(o2)
# Test nested extended attributes
ptr2.setExtended('outputImages.7')
self.assertEqual(imgSet[7], ptr2.get())
# Same as ptr2, but setting extended in constructor
ptr3 = pwobj.Pointer(value=o2, extended='outputImages.7')
self.assertEqual(imgSet[7], ptr3.get())
# Test copy between pointer objects
ptr4 = pwobj.Pointer()
ptr4.copy(ptr3)
self.assertEqual(imgSet[7], ptr4.get())
self.assertEqual(ptr4.getExtended(), 'outputImages.7')
