def process(tr, parameters, tableBuilder):
ids = parameters.get("identifiers")
search_service = tr.getSearchService()
expCodes = []
if "Experiment" in parameters:
print "preparing experiment update"
for exp in search_service.listExperiments(parameters.get("Project")):
expCodes.append(exp.getExperimentIdentifier().split("/")[-1])
for id in ids:
print "searching id "+id
entity = None
if "Experiment" in parameters and id in expCodes:
entity = tr.getExperimentForUpdate(parameters.get("Project")+"/"+id)
else:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, id))
found = search_service.searchForSamples(sc)
print "found: "+str(found)
if len(found) > 0:
entity = tr.getSampleForUpdate(found[0].getSampleIdentifier())
if entity:
for type in parameters.get("types"):
print "handling type "+type
typeMap = parameters.get(type)
print typeMap
try:
value = typeMap.get(id)
print "value "+value
entity.setPropertyValue(type,value)
except:
print "exception when trying to set property value!"
pass
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.PROJECT, project))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
foundSamples = search_service.searchForSamples(sc)
foundSamplesFilter = [
s for s in foundSamples if 'ENTITY' not in s.getCode()
]
offset = 0
exists = True
while exists:
# create new barcode
newBarcode = getNextFreeBarcode(
project,
len(foundSamplesFilter) + len(newTestSamples) + offset)
# check if barcode already exists in database
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, newBarcode))
found = search_service.searchForSamples(pc)
if len(found) == 0:
exists = False
else:
offset += 1
return newBarcode
def _getExperimentSample(collectionPermId, expSamplePermId):
"""Retrieve the experiment sample."""
# Get the experiment sample
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
expSamplePermId)
)
expCriteria = SearchCriteria()
expCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
collectionPermId)
)
# Add the experiment subcriteria
sampleCriteria.addSubCriteria(
SearchSubCriteria.createExperimentCriteria(
expCriteria)
)
# Search
expSampleList = searchService.searchForSamples(sampleCriteria)
if len(expSampleList) != 1:
return None
# Return the experiment sample
return expSampleList[0]
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if key == None:
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName " + name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)
)
foundSamples = ss.searchForSamples(sc)
samplehit = foundSamples[0]
sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier())
parents = samplehit.getParentSampleIdentifiers()
parentcodes = []
for parent in parents:
parentcodes.append(parent.split("/")[-1])
parentInfos = "_".join(parentcodes)
experiment = transaction.getExperimentForUpdate("/" + space + "/" + project + "/" + experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(time.time()).strftime("%Y-%m-%d %H:%M:%S")
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
# Register files
dataSetRes = transaction.createNewDataSet("Q_WF_NGS_RNAEXPRESSIONANALYSIS_RESULTS")
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet("Q_WF_NGS_RNAEXPRESSIONANALYSIS_LOGS")
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
resultsname = incomingPath + "/" + parentInfos + "_workflow_results"
logname = incomingPath + "/" + parentInfos + "_workflow_logs"
os.rename(incomingPath + "/logs", logname)
os.rename(incomingPath + "/result", resultsname)
transaction.moveFile(resultsname, dataSetRes)
transaction.moveFile(logname, dataSetLogs)
def _getDataSetForTube(self, tubeCode=None):
"""
Get the datasets belonging to the tube with specified tube code.
If none is found, return [].
If no tubeCode is given, it is assumed that the tube is the passed
entity with code self._entityCode.
"""
if tubeCode is None:
tubeCode = self._entityCode
# Set search criteria to retrieve the dataset contained in the tube
searchCriteria = SearchCriteria()
tubeCriteria = SearchCriteria()
tubeCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, tubeCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(tubeCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for tube " \
"with code " + tubeCode + "."
self._logger.error(self._message)
# Return
return dataSets
def _getAllTubes(self):
"""
Get all tubes in the experiment. If the specimen is set (self._specimen),
then return only those tubes that belong to it.
Returns [] if none are found.
"""
# Set search criteria to retrieve all tubes in the experiment
# All tubes belong to a virtual tubeset - so the set of tubes in the
# experiment is exactly the same as the set of tubes in the virtual
# tubeset
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, self._experimentPrefix + "_TUBE"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
tubes = searchService.searchForSamples(searchCriteria)
if len(tubes) == 0:
self._message = "The experiment with code " + \
self._experimentCode + "does not contain tubes."
self._logger.error(self._message)
return tubes
# Check that the specimen matches (if needed)
if self._specimen != "":
tubes = [tube for tube in tubes if \
tube.getPropertyValue(self._experimentPrefix + "_SPECIMEN") == self._specimen]
# Return the (filtered) tubes
return tubes
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
# Get the name of the incoming file
name = transaction.getIncoming().getName()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
identifier = pattern.findall(name)[0]
code = None
code = identifier[:10]
# Find the test sample
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("EXPRESSION_MATRIX")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
search_service = transaction.getSearchService()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
for name in os.listdir(incomingPath):
identifier = None
searchID = pattern.findall(name)
if isExpected(searchID[0]):
identifier = searchID[0]
project = identifier[:5]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, "MA"+identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MA_CHIP_IMAGE")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
image = os.path.realpath(os.path.join(incomingPath,name))
transaction.moveFile(image, dataSet)
def _getDataSetsForSample(sampleIdentifier, dataSetType):
"""Return the dataSet of given type for specified sample."""
# Set search criteria to retrieve the dataSet associated with the sample
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
dataSetType)
)
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.CODE,
sampleIdentifier)
)
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(
sampleCriteria)
)
dataSetList = searchService.searchForDataSets(searchCriteria)
if len(dataSetList) != 1:
[]
# Return the dataSet
return dataSetList
def _retrieveSampleWithTypeAndPermId(self, samplePermId, sampleType):
"""
Retrieve a sample belonging to current experiment
sample and collection having specified type and perm id.
"""
# The sample is of type 'sampleType' and has id 'sampleId'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
sampleType))
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
samplePermId))
# Now search
samples = searchService.searchForSamples(searchCriteria)
if len(samples) != 1:
self._message = "Sample with id " + sampleId + \
" and type " + sampleType + "not found!"
self._logger.error(self._message)
return None
# Return the sample
return samples[0]
def _getDataSetForWell(self, wellId=None):
"""
Get the datasets belonging to the well with specified code. If none
are found, return [].
If no wellId is given, it is assumed that the well is the passed
entity with code self._entityId.
"""
if wellId is None:
wellId = self._entityId
# Set search criteria to retrieve the dataset contained in the well
searchCriteria = SearchCriteria()
wellCriteria = SearchCriteria()
wellCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE,
wellId))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(wellCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for well " \
"with code " + wellId + "."
self._logger.error(self._message)
# Return
return dataSets
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
name = transaction.getIncoming().getName()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
#name = transaction.getIncoming().getName()
parents = []
identifier = pattern.findall(name)[0]
if isExpected(identifier):
experiment = identifier[1:5]
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, parentCode))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(parentSampleIdentifier)
# register new experiment and sample
existingExperimentIDs = []
existingExperiments = search_service.listExperiments("/" + space + "/" + project)
numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
for eexp in existingExperiments:
existingExperimentIDs.append(eexp.getExperimentIdentifier())
newExpID = '/' + space + '/' + project + '/' + project + 'E' +str(numberOfExperiments)
while newExpID in existingExperimentIDs:
numberOfExperiments += 1
newExpID = '/' + space + '/' + project + '/' + project + 'E' +str(numberOfExperiments)
newHLATypingExperiment = transaction.createNewExperiment(newExpID, "Q_NGS_HLATYPING")
newHLATypingExperiment.setPropertyValue('Q_CURRENT_STATUS', 'FINISHED')
newHLATypingSample = transaction.createNewSample('/' + space + '/' + 'HLA'+ parentCode, "Q_NGS_HLATYPING")
newHLATypingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
newHLATypingSample.setExperiment(newHLATypingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("Q_NGS_HLATYPING_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(newHLATypingSample)
transaction.moveFile(incomingPath, dataSet)
def _getDataSetForTube(self, tubeCode):
"""
Get the datasets belonging to the tube with specified tube code.
If none is found, return [].
"""
if _DEBUG:
self._logger.info("Searching for tube with code " + tubeCode)
# Set search criteria to retrieve the dataset contained in the tube
searchCriteria = SearchCriteria()
tubeCriteria = SearchCriteria()
tubeCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE,
tubeCode))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(tubeCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if _DEBUG:
self._logger.info("Retrieved " + str(len(dataSets)) + \
" datasets for tube with code " + tubeCode)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for tube " \
"with code " + tubeCode + "."
self._logger.error(self._message)
# Return
return dataSets
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
search_service = transaction.getSearchService()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
for name in os.listdir(incomingPath):
identifier = None
searchID = pattern.findall(name)
if isExpected(searchID[0]):
identifier = searchID[0]
project = identifier[:5]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, "MA" + identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MA_CHIP_IMAGE")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
image = os.path.realpath(os.path.join(incomingPath, name))
transaction.moveFile(image, dataSet)
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.PROJECT, project));
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
foundSamples = search_service.searchForSamples(sc)
foundSamplesFilter = [s for s in foundSamples if 'ENTITY' not in s.getCode()]
offset = 0
exists = True
while exists:
# create new barcode
newBarcode = getNextFreeBarcode(project, len(foundSamplesFilter) + len(newTestSamples) + offset)
# check if barcode already exists in database
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, newBarcode))
found = search_service.searchForSamples(pc)
if len(found) == 0:
exists = False
else:
offset += 1
return newBarcode
def _getAllTubes(self):
"""
Get all tubes in the experiment. If the specimen is set (self._specimen),
then return only those tubes that belong to it.
Returns [] if none are found.
"""
# Set search criteria to retrieve all tubes in the experiment
# All tubes belong to a virtual tubeset - so the set of tubes in the
# experiment is exactly the same as the set of tubes in the virtual
# tubeset
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, self._experimentPrefix + "_TUBE"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
tubes = searchService.searchForSamples(searchCriteria)
if len(tubes) == 0:
self._message = "Could not retrieve tubes for experiment with code " + self._experimentCode + "."
return tubes
# Check that the specimen matches (if needed)
if self._specimen != "":
tubes = [tube for tube in tubes if \
tube.getPropertyValue(self._experimentPrefix + "_SPECIMEN") == self._specimen]
# Return the (filtered) tubes
return tubes
def _getDataSetForTube(self, tubeCode=None):
"""
Get the datasets belonging to the tube with specified tube code.
If none is found, return [].
If no tubeCode is given, it is assumed that the tube is the passed
entity with code self._entityCode.
"""
if tubeCode is None:
tubeCode = self._entityCode
# Set search criteria to retrieve the dataset contained in the tube
searchCriteria = SearchCriteria()
tubeCriteria = SearchCriteria()
tubeCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, tubeCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(tubeCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for tube " \
"with code " + tubeCode + "."
# Return
return dataSets
def _getDataSetForWell(self, wellCode=None):
"""
Get the datasets belonging to the well with specified code. If none
are found, return [].
If no wellCode is given, it is assumed that the well is the passed
entity with code self._entityCode.
"""
if wellCode is None:
wellCode = self._entityCode
# Set search criteria to retrieve the dataset contained in the well
searchCriteria = SearchCriteria()
wellCriteria = SearchCriteria()
wellCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, wellCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(wellCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for well " \
"with code " + wellCode + "."
# Return
return dataSets
def process(tr, params, tableBuilder):
if "user" in params:
tr.setUserId(params.get("user"))
for sample in params.keySet():
parameters = params.get(sample)
sampleCode = parameters.get("code")
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = search_service.searchForSamples(sc)
if(foundSamples.size() < 1):
proj = parameters.get("project")
space = parameters.get("space")
sampleType = parameters.get("type")
species = parameters.get("species")
sampleId = "/" + space + "/" + sampleCode
sample = tr.createNewSample(sampleId, sampleType)
exp = "/"+space+"/"+proj+"/"+parameters.get("experiment")
exp = tr.getExperiment(exp)
sample.setExperiment(exp)
if parameters.get("Q_SECONDARY_NAME"):
sample.setPropertyValue("Q_SECONDARY_NAME",parameters.get("Q_SECONDARY_NAME"))
if parameters.get("parents"):
sample.setParentSampleIdentifiers(parameters.get("parents"))
if parameters.get("metadata"):
properties = parameters.get("metadata")
for prop in properties.keySet():
sample.setPropertyValue(prop, properties.get(prop))
def _getFlowExperimentSample(self):
"""Find the {FLOW}_EXPERIMENT sample with given Id."""
# Inform
if _DEBUG:
self._logger.info("Retrieving experiment sample of code " + \
self._expSampleId + ", permId " + self._expSamplePermId + \
" and type " + self._expSampleType)
# Search sample of type MICROSCOPY_EXPERIMENT with specified CODE
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
self._expSampleType))
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
# Search
samples = searchService.searchForSamples(sampleCriteria)
if len(samples) == 0:
samples = []
self._message = "Could not retrieve " + self._expSampleType + " sample with code " + \
self._expSampleId + ", permId " + self._expSamplePermId + \
" and type " + self._expSampleType + "."
self._logger.error(self._message)
return samples
if _DEBUG:
self._logger.info("Successfully returned sample " +
self._expSampleId)
# Return
return samples[0]
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
#sample_id = experiment_id+'.'
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName " + name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = ss.searchForSamples(sc)
samplehit = foundSamples[0]
sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier())
experiment = transaction.getExperimentForUpdate("/" + space + "/" +
project + "/" +
experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
#Register files
dataSetRes = transaction.createNewDataSet('Q_NGS_READ_MATCH_ARCHIVE')
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet(
'Q_WF_NGS_16S_TAXONOMIC_PROFILING_LOGS')
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
resultsname = incomingPath + "/" + experiment_id + "_workflow_results"
logname = incomingPath + "/" + experiment_id + "_workflow_logs"
os.rename(incomingPath + "/logs", logname)
os.rename(incomingPath + "/result", resultsname)
transaction.moveFile(resultsname, dataSetRes)
transaction.moveFile(logname, dataSetLogs)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if key == None:
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
# parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, identifier)
)
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
# newVariantCallingExperiment = transaction.createNewExperiment('/' + space + '/' + project + '/' + project + 'E' + str(numberOfExperiments), "Q_NGS_VARIANT_CALLING")
# newVariantCallingSample = transaction.createNewSample('/' + space + '/' + 'VC'+ parentCode, "Q_NGS_VARIANT_CALLING")
# newVariantCallingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
# newVariantCallingSample.setExperiment(newVariantCallingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("FEATUREXML")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
# cegat = False
f = "source_dropbox.txt"
sourceLabFile = open(os.path.join(incomingPath, f))
sourceLab = sourceLabFile.readline().strip()
sourceLabFile.close()
# if sourceLab == 'dmcegat':
# cegat = True
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
for f in os.listdir(incomingPath):
if ".testorig" in f:
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
# elif f.endswith('vcf') and cegat:
# secondaryName = f.split('_')[0]
# entitySample = transaction.getSampleForUpdate('/%s/%s' % (space,parentCode))
# sa.setPropertyValue('Q_SECONDARY_NAME', secondaryName)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
#parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
#numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
#newVariantCallingExperiment = transaction.createNewExperiment('/' + space + '/' + project + '/' + project + 'E' + str(numberOfExperiments), "Q_NGS_VARIANT_CALLING")
#newVariantCallingSample = transaction.createNewSample('/' + space + '/' + 'VC'+ parentCode, "Q_NGS_VARIANT_CALLING")
#newVariantCallingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
#newVariantCallingSample.setExperiment(newVariantCallingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("IDXML")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
#cegat = False
f = "source_dropbox.txt"
sourceLabFile = open(os.path.join(incomingPath, f))
sourceLab = sourceLabFile.readline().strip()
sourceLabFile.close()
#if sourceLab == 'dmcegat':
#cegat = True
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
for f in os.listdir(incomingPath):
if ".testorig" in f:
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
#elif f.endswith('vcf') and cegat:
#secondaryName = f.split('_')[0]
#entitySample = transaction.getSampleForUpdate('/%s/%s' % (space,parentCode))
#sa.setPropertyValue('Q_SECONDARY_NAME', secondaryName)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, identifier))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(parentSampleIdentifier)
# find or register new experiment
expType = "Q_MS_MEASUREMENT"
msExperiment = None
experiments = search_service.listExperiments("/" + space + "/" + project)
experimentIDs = []
for exp in experiments:
experimentIDs.append(exp.getExperimentIdentifier())
if exp.getExperimentType() == expType:
msExperiment = exp
# no existing experiment for samples of this sample preparation found
if not msExperiment:
expID = experimentIDs[0]
i = 0
while expID in experimentIDs:
i += 1
expNum = len(experiments) + i
expID = '/' + space + '/' + project + '/' + project + 'E' + str(expNum)
msExperiment = transaction.createNewExperiment(expID, expType)
newMSSample = transaction.createNewSample('/' + space + '/' + 'MS'+ parentCode, "Q_MS_RUN")
newMSSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
newMSSample.setExperiment(msExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MS_MZML_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(newMSSample)
transaction.moveFile(incomingPath, dataSet)
def sampleExists(searchService, sampleCode):
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = searchService.searchForSamples(sc)
if len(foundSamples) > 0:
return True
return False
def get_space_from_project(transaction, project):
search_service = transaction.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.PROJECT, project))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
foundSamples = search_service.searchForSamples(sc)
space = foundSamples[0].getSpace()
return space
def get_dataset_for_permid(transaction, permid):
search_service = transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, permid))
found = list(search_service.searchForDataSets(criteria))
if len(found) == 1:
return found[0]
else:
return None
def runs(self):
"""Return *all* runs in the db.
TODO this is madness!! At least, this should only return
runs in this project.
"""
search = self.transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, 'Q_MS_RUN'))
return search.searchForSamples(criteria)
def process(tr, parameters, tableBuilder):
#ids = sorted(parameters.get("ids"))
types = parameters.get(
"types") #sample types (tiers) that are requested for the tsv
project = parameters.get("project")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
#search all samples of project
search = tr.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.PROJECT, project))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
fetchOptions = EnumSet.of(SampleFetchOption.ANCESTORS,
SampleFetchOption.PROPERTIES)
allSamples = search.searchForSamples(sc, fetchOptions)
#filter all samples by types
samples = []
for s in allSamples:
if s.getSampleType() in types:
samples.append(s)
#sort remaining samples-
samples = sorted(samples)
voc = search.getVocabulary("Q_NCBI_TAXONOMY")
for s in samples:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME,
sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource([sample], voc.getTerms(), []))
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType == "CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def get_dataset_for_name(transaction, dataset_name):
search_service = transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
MatchClause.createPropertyMatch('NAME', dataset_name))
found = list(search_service.searchForDataSets(criteria))
if len(found) == 1:
print("DataSetCode of found dataset = " + found[0].getDataSetCode())
return transaction.getDataSetForUpdate(found[0].getDataSetCode())
#return found[0]
else:
return None
def listSamplesForExperiment(searchService, sampleType, expID):
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, sampleType))
ec = SearchCriteria()
ec.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, expID))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(ec))
existingSamples = searchService.searchForSamples(sc)
return existingSamples
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(PARENT)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code)
)
sample = searchService.searchForSamples(sc)[0]
handleSample(sample, tableBuilder)
def isCurrentMSRun(tr, parentExpID, msExpID):
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.TYPE, "Q_MS_RUN")
)
foundSamples = search_service.searchForSamples(sc)
for samp in foundSamples:
currentMSExp = samp.getExperiment()
if currentMSExp.getExperimentIdentifier() == msExpID:
for parID in samp.getParentSampleIdentifiers():
parExp = tr.getSampleForUpdate(parID).getExperiment().getExperimentIdentifier()
if parExp == parentExpID:
return True
return False
def runs(self):
"""Return *all* runs in the db.
TODO this is madness!! At least, this should only return
runs in this project.
"""
search = self.transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE,
'Q_MS_RUN'
)
)
return search.searchForSamples(criteria)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = ss.searchForSamples(sc)
sample = foundSamples[0]
sample = transaction.getSampleForUpdate(sample.getSampleIdentifier())
experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
#Register files
dataSetRes = transaction.createNewDataSet('Q_WF_EDDA_BENCHMARK_RESULTS')
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet('Q_WF_EDDA_BENCHMARK_LOGS')
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
transaction.moveFile(incomingPath+"/result", dataSetRes)
transaction.moveFile(incomingPath+"/logs", dataSetLogs)
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
offset = 0
exists = True
while exists:
n = str(len(newTestSamples)+1+offset) #for future programmers: sorry if this reaches > 999 !
code = project+n.zfill(3)+"X" # should go to other letters in that case
code = code+checksum.checksum(code)
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
found = search_service.searchForSamples(pc)
print found
if len(found) == 0:
exists = False
else:
offset += 1
return code
def isCurrentMSRun(tr, parentExpID, msExpID):
"""Ask Andreas"""
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, "Q_MS_RUN"))
foundSamples = search_service.searchForSamples(sc)
for samp in foundSamples:
currentMSExp = samp.getExperiment()
if currentMSExp.getExperimentIdentifier() == msExpID:
for parID in samp.getParentSampleIdentifiers():
parExp = (tr.getSampleForUpdate(
parID).getExperiment().getExperimentIdentifier())
if parExp == parentExpID:
return True
return False
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
code = pattern.findall(name)[0]
#such checks should not be needed here as it's not raw data
#if isExpected(code):
# project = code[2:7]
# #parentCode = code[2:]
#else:
# print "The identifier "+code+" did not match the pattern MSQ[A-Z]{4}\d{3}\w{2}"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, "MS"+code))
# wait for registration of ms sample to finish (done in raw dropbox after mzml marker is created)
timeout = 10
while timeout > 0:
timeout = timeout - 1
time.sleep(5)
foundSamples = search_service.searchForSamples(sc)
if foundSamples > 0:
break
sampleIdentifier = foundSamples[0].getSampleIdentifier()
#space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MS_RAW_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
space = "DEFAULT" #will be set later
project = "DEFAULT"
experiment = "DEFAULT"
sample_id = "DEFAULT"
extension = name.split(".")[-1]
identifier = pattern.findall(name)[0]
if isExpected(identifier, extension):
print "found identifier"
#experiment = identifier[1:5]
project = identifier[4:9]
sample_id = identifier[:14]
else:
print "The identifier "+identifier+" did not match the pattern MSQCQ[A-Z]{4}\d{3}\w{2}, the checksum was wrong or the file is not of type ."+ext
#Register file
dataSet = transaction.createNewDataSet('Q_WF_MS_QUALITYCONTROL_RESULTS')
dataSet.setMeasuredData(False)
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sample_id))
foundSamples = search_service.searchForSamples(sc)
if foundSamples.size() > 0:
sa = transaction.getSampleForUpdate(foundSamples[0].getSampleIdentifier())
transaction.moveFile(incomingPath, dataSet)
dataSet.setSample(sa)
else:
#Sample not found, something went wrong and the file is attached to an "Unknown" sample
transaction.moveFile(incomingPath, dataSet)
newSample = transaction.createNewSample('/' + space + '/' + sample_id, 'UNKNOWN')
newSample.setExperiment(experiment)
dataSet.setSample(newSample)
def process(tr, parameters, tableBuilder):
ids = sorted(parameters.get("ids"))
types = parameters.get(
"types") #sample types (tiers) that are requested for the tsv
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
sampleMap = {}
voc = searchService.getVocabulary("Q_NCBI_TAXONOMY")
for id in ids:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE,
id.split('/')[-1]))
sample = searchService.searchForSamples(sc)[0]
sampleMap[id] = sample
for id in ids:
sample = sampleMap[id]
if sample.getSampleType() in types:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME,
sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource(id, sampleMap, voc.getTerms()))
row.setCell(EXTERNAL_ID,
sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType == "CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(XML)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
sample = searchService.searchForSamples(sc)[0]
row = tableBuilder.addRow()
row.setCell(CODE, sample.getCode())
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
def getsample(qcode, transaction):
"""Resturns a immutable sample object for a given
QBIC barcode."""
sserv = transaction.getSearchService()
scrit = SearchCriteria()
scrit.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, qcode))
result = sserv.searchForSamples(scrit)
if not result:
raise mtbutils.MTBdropboxerror(
'No matching sample found in openBIS for code {}.'.format(qcode))
if len(result) > 1:
raise mtbutils.MTBdropboxerror(
'More than one sample found in openBIS for code {}.'.format(qcode))
return result[0]
def _getDataSetsForPlate(self, plateCode=None):
"""
Return a list of datasets belonging to the plate with specified ID
optionally filtered by self._specimen. If none are found, return [].
If no plateCode is given, it is assumed that the plate is the passed
entity with code self._entityCode.
"""
if plateCode is None:
plateCode = self._entityCode
# Set search criteria to retrieve all wells contained in the plate
searchCriteria = SearchCriteria()
plateCriteria = SearchCriteria()
plateCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, plateCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleContainerCriteria(plateCriteria))
wells = searchService.searchForSamples(searchCriteria)
if len(wells) == 0:
self._message = "Could not retrieve wells for plate with " \
"code " + plateCode + "."
self._logger.error(self._message)
return wells
# Check that the specimen matches (if needed)
if self._specimen != "":
wells = [well for well in wells if \
well.getPropertyValue(self._experimentPrefix + "_SPECIMEN") == self._specimen]
# Now iterate over the samples and retrieve their datasets
dataSets = []
for well in wells:
wellCode = well.getCode()
dataSetsForWell = self._getDataSetForWell(wellCode)
dataSets.extend(dataSetsForWell)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for wells in " \
"plate with code " + plateCode + " from experiment " \
"with code " + self._experimentCode + "."
self._logger.error(self._message)
# Return
return dataSets
def _getAllPlates(self):
"""
Get all plates in the experiment. Returns [] if none are found.
"""
# Set search criteria to retrieve all plates in the experiment
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, self._experimentPrefix + "_PLATE"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
plates = searchService.searchForSamples(searchCriteria)
if len(plates) == 0:
self._message = "Could not retrieve plates for experiment with code " + self._experimentCode + "."
return plates
# Return the plates
return plates
def _getMicroscopySampleTypeSample(self):
# Search sample of type MICROSCOPY_SAMPLE_TYPE with specified CODE
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
self._sampleType)
)
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
self._samplePermId)
)
# Search parent sample of type MICROSCOPY_EXPERIMENT with specified permId
sampleParentCriteria = SearchCriteria()
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
self._expSampleType))
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
# Add the parent sample subcriteria
sampleCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(
sampleParentCriteria
)
)
# Search
samples = searchService.searchForSamples(sampleCriteria)
if len(samples) == 0:
samples = []
self._message = "Could not retrieve MICROSCOPY_SAMPLE_TYPE sample with id " + \
self._sampleId + " for parent sample MICROSCOPY_EXPERIMENT with id " + \
self._expSampleId + " from COLLECTION experiment " + self._collectionId + "."
self._logger.error(self._message)
return samples
if _DEBUG:
self._logger.info("Retrieved " + str(len(samples)) + \
" samples of type MICROSCOPY_SAMPLE_TYPE " + \
"for parent sample MICROSCOPY_EXPERIMENT " +
"with ID " + self._expSamplePermId)
# Return
return samples[0]
def _getMicroscopySampleTypeSample(self):
# Search sample of type MICROSCOPY_SAMPLE_TYPE with specified CODE
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
self._sampleType)
)
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
self._samplePermId)
)
# Search parent sample of type MICROSCOPY_EXPERIMENT with specified permId
sampleParentCriteria = SearchCriteria()
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
self._expSampleType))
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
# Add the parent sample subcriteria
sampleCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(
sampleParentCriteria
)
)
# Search
samples = searchService.searchForSamples(sampleCriteria)
if len(samples) == 0:
samples = []
self._message = "Could not retrieve MICROSCOPY_SAMPLE_TYPE sample with id " + \
self._sampleId + " for parent sample MICROSCOPY_EXPERIMENT with id " + \
self._expSampleId + " from COLLECTION experiment " + self._collectionId + "."
self._logger.error(self._message)
return samples
if _DEBUG:
self._logger.info("Retrieved " + str(len(samples)) + \
" samples of type MICROSCOPY_SAMPLE_TYPE " + \
"for parent sample MICROSCOPY_EXPERIMENT " +
"with ID " + self._expSamplePermId)
# Return
return samples[0]
def _bio_sample(self, barcode):
"""Find a sample in openbis by it's barcode.
Since we use the barcode an unique identifier, there should never
be more than one sample with a given barcode.
"""
search = self._transaction.getSearchService()
criteria = SearchCriteria()
barcode_match = SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, barcode)
criteria.addMatchClause(barcode_match)
samples = search.searchForSamples(criteria)
if len(samples) > 1:
raise RuntimeError("Found more than one sample for barcode %s." %
barcode)
if not samples:
raise ValueError("Could not find a sample for barcode %s" %
barcode)
sample = samples[0]
return sample.getSpace(), self.barcode[:5], sample
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(XML)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code)
)
sample = searchService.searchForSamples(sc)[0]
row = tableBuilder.addRow()
row.setCell(CODE, sample.getCode())
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
offset = 0
exists = True
while exists:
n = str(len(newTestSamples) + 1 +
offset) #for future programmers: sorry if this reaches > 999 !
code = project + n.zfill(
3) + "X" # should go to other letters in that case
code = code + checksum.checksum(code)
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, code))
found = search_service.searchForSamples(pc)
print found
if len(found) == 0:
exists = False
else:
offset += 1
return code
def _getChildSamples(parentSampleType, parentSamplePermId, sampleType):
"""Return the samples of given type for specified parent sample."""
# The samples are of type 'sampleType'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
sampleType))
# The samples have given parent
expSampleCriteria = SearchCriteria()
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
parentSampleType))
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
parentSamplePermId))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(expSampleCriteria))
# Now search
samples = searchService.searchForSamples(searchCriteria)
# Return the samples
return samples
def _retrieveAllSamplesWithTypeAndParent(self, sampleType, parentSampleId,
parentSampleType):
"""
Retrieve all samples belonging to current experiment
sample and collection having specified type.
"""
if _DEBUG:
self._logger.info("Retrieving samples of type " + sampleType +
" with parent sample with id " + parentSampleId +
" and type " + parentSampleType)
# The samples are of type 'sampleType'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
sampleType))
# The samples have given parent
expSampleCriteria = SearchCriteria()
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
parentSampleType))
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE,
parentSampleId))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(expSampleCriteria))
# Now search
samples = searchService.searchForSamples(searchCriteria)
# Return the samples
return samples
def _getChildSamples(parentSampleType, parentSamplePermId, sampleType):
"""Return the samples of given type for specified parent sample."""
# The samples are of type 'sampleType'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
sampleType)
)
# The samples have given parent
expSampleCriteria = SearchCriteria()
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
parentSampleType)
)
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
parentSamplePermId)
)
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(expSampleCriteria)
)
# Now search
samples = searchService.searchForSamples(searchCriteria)
# Return the samples
return samples
def _retrieveAllSamplesWithType(self, sampleType):
"""
Retrieve all samples belonging to current experiment
sample and collection having specified type.
"""
# The samples are of type 'sampleType'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
sampleType))
# The samples have parent _EXPERIMENT_SAMPLE
expSampleCriteria = SearchCriteria()
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
self._expSampleType))
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(expSampleCriteria))
# Now search
samples = searchService.searchForSamples(searchCriteria)
# Return the samples
return samples
def _getDataSetsForExperiment(self):
"""
Return a list of datasets belonging to the experiment and optionally
to the sample. If the sample ID is empty, only the experiment is used
in the search criteria.
If none are found, return [].
"""
# Set search criteria to retrieve all datasets for the experiment.
# If the sample code is set, we also filter by it.
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, "MICROSCOPY_IMG_CONTAINER"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
if self._sample is not None:
self._logger.info("Filter by sample " + self._sampleId)
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._sample.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(sampleCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
dataSets = []
self._message = "Could not retrieve datasets for experiment " \
"with id " + self._experimentId
if self._sampleId != "":
self._message = self._message + " and sample with id " + \
self._sampleId
self._logger.error(self._message)
# Return
return dataSets
def process(tr, parameters, tableBuilder):
ids = sorted(parameters.get("ids"))
types = parameters.get("types") #sample types (tiers) that are requested for the tsv
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
sampleMap = {}
voc = searchService.getVocabulary("Q_NCBI_TAXONOMY")
for id in ids:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, id.split('/')[-1]))
sample = searchService.searchForSamples(sc)[0]
sampleMap[id] = sample
for id in ids:
sample = sampleMap[id]
if sample.getSampleType() in types:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource(id, sampleMap, voc.getTerms()))
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType=="CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def _bio_sample(self, barcode):
"""Find a sample in openbis by it's barcode.
Since we use the barcode an unique identifier, there should never
be more than one sample with a given barcode.
"""
search = self._transaction.getSearchService()
criteria = SearchCriteria()
barcode_match = SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, barcode
)
criteria.addMatchClause(barcode_match)
samples = search.searchForSamples(criteria)
if len(samples) > 1:
raise RuntimeError(
"Found more than one sample for barcode %s." % barcode
)
if not samples:
raise ValueError(
"Could not find a sample for barcode %s" % barcode
)
sample = samples[0]
return sample.getSpace(), self.barcode[:5], sample
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
experiment = identifier[1:5]
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, parentCode))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
parentSample = transaction.getSampleForUpdate(parentSampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_PEPTIDE_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(parentSample)
transaction.moveFile(incomingPath, dataSet)
