def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
# Get the name of the incoming file
name = transaction.getIncoming().getName()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
identifier = pattern.findall(name)[0]
code = None
code = identifier[:10]
# Find the test sample
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("EXPRESSION_MATRIX")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
transaction.moveFile(incomingPath, dataSet)
def process(tr, params, tableBuilder):
if "user" in params:
tr.setUserId(params.get("user"))
for sample in params.keySet():
parameters = params.get(sample)
sampleCode = parameters.get("code")
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = search_service.searchForSamples(sc)
if(foundSamples.size() < 1):
proj = parameters.get("project")
space = parameters.get("space")
sampleType = parameters.get("type")
species = parameters.get("species")
sampleId = "/" + space + "/" + sampleCode
sample = tr.createNewSample(sampleId, sampleType)
exp = "/"+space+"/"+proj+"/"+parameters.get("experiment")
exp = tr.getExperiment(exp)
sample.setExperiment(exp)
if parameters.get("Q_SECONDARY_NAME"):
sample.setPropertyValue("Q_SECONDARY_NAME",parameters.get("Q_SECONDARY_NAME"))
if parameters.get("parents"):
sample.setParentSampleIdentifiers(parameters.get("parents"))
if parameters.get("metadata"):
properties = parameters.get("metadata")
for prop in properties.keySet():
sample.setPropertyValue(prop, properties.get(prop))
def _getAllTubes(self):
"""
Get all tubes in the experiment. If the specimen is set (self._specimen),
then return only those tubes that belong to it.
Returns [] if none are found.
"""
# Set search criteria to retrieve all tubes in the experiment
# All tubes belong to a virtual tubeset - so the set of tubes in the
# experiment is exactly the same as the set of tubes in the virtual
# tubeset
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, self._experimentPrefix + "_TUBE"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
tubes = searchService.searchForSamples(searchCriteria)
if len(tubes) == 0:
self._message = "The experiment with code " + \
self._experimentCode + "does not contain tubes."
self._logger.error(self._message)
return tubes
# Check that the specimen matches (if needed)
if self._specimen != "":
tubes = [tube for tube in tubes if \
tube.getPropertyValue(self._experimentPrefix + "_SPECIMEN") == self._specimen]
# Return the (filtered) tubes
return tubes
def _getChildSamples(parentSampleType, parentSamplePermId, sampleType):
"""Return the samples of given type for specified parent sample."""
# The samples are of type 'sampleType'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
sampleType)
)
# The samples have given parent
expSampleCriteria = SearchCriteria()
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
parentSampleType)
)
expSampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
parentSamplePermId)
)
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(expSampleCriteria)
)
# Now search
samples = searchService.searchForSamples(searchCriteria)
# Return the samples
return samples
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, identifier))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(parentSampleIdentifier)
# find or register new experiment
expType = "Q_MS_MEASUREMENT"
msExperiment = None
experiments = search_service.listExperiments("/" + space + "/" + project)
experimentIDs = []
for exp in experiments:
experimentIDs.append(exp.getExperimentIdentifier())
if exp.getExperimentType() == expType:
msExperiment = exp
# no existing experiment for samples of this sample preparation found
if not msExperiment:
expID = experimentIDs[0]
i = 0
while expID in experimentIDs:
i += 1
expNum = len(experiments) + i
expID = '/' + space + '/' + project + '/' + project + 'E' + str(
expNum)
msExperiment = transaction.createNewExperiment(expID, expType)
newMSSample = transaction.createNewSample(
'/' + space + '/' + 'MS' + parentCode, "Q_MS_RUN")
newMSSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
newMSSample.setExperiment(msExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MS_MZML_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(newMSSample)
transaction.moveFile(incomingPath, dataSet)
def process(tr, parameters, tableBuilder):
print parameters
if "user" in parameters:
tr.setUserId(parameters.get("user"))
sampleCode = parameters.get("code")
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = search_service.searchForSamples(sc)
if(foundSamples.size() < 1):
proj = parameters.get("project")
space = parameters.get("space")
sampleType = parameters.get("type")
sampleId = "/" + space + "/" + sampleCode
sample = tr.createNewSample(sampleId, sampleType)
exp = "/"+space+"/"+proj+"/"+parameters.get("experiment")
exp = tr.getExperiment(exp)
sample.setExperiment(exp)
if parameters.get("sample_class"):
sample.setPropertyValue("Q_SECONDARY_NAME",parameters.get("sample_class"))
if parameters.get("parents"):
sample.setParentSampleIdentifiers(parameters.get("parents"))
if parameters.get("properties"):
properties = parameters.get("properties")
for prop in properties.keySet():
sample.setPropertyValue(prop, properties.get(prop))
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
search_service = transaction.getSearchService()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
for name in os.listdir(incomingPath):
identifier = None
searchID = pattern.findall(name)
if isExpected(searchID[0]):
identifier = searchID[0]
project = identifier[:5]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, "MA" + identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MA_CHIP_IMAGE")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
image = os.path.realpath(os.path.join(incomingPath, name))
transaction.moveFile(image, dataSet)
def process(tr, parameters, tableBuilder):
ids = parameters.get("identifiers")
search_service = tr.getSearchService()
expCodes = []
if "Experiment" in parameters:
print "preparing experiment update"
for exp in search_service.listExperiments(parameters.get("Project")):
expCodes.append(exp.getExperimentIdentifier().split("/")[-1])
for id in ids:
print "searching id "+id
entity = None
if "Experiment" in parameters and id in expCodes:
entity = tr.getExperimentForUpdate(parameters.get("Project")+"/"+id)
else:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, id))
found = search_service.searchForSamples(sc)
print "found: "+str(found)
if len(found) > 0:
entity = tr.getSampleForUpdate(found[0].getSampleIdentifier())
if entity:
for type in parameters.get("types"):
print "handling type "+type
typeMap = parameters.get(type)
print typeMap
try:
value = typeMap.get(id)
print "value "+value
entity.setPropertyValue(type,value)
except:
print "exception when trying to set property value!"
pass
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
search_service = transaction.getSearchService()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
for name in os.listdir(incomingPath):
identifier = None
searchID = pattern.findall(name)
if isExpected(searchID[0]):
identifier = searchID[0]
project = identifier[:5]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, "MA"+identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MA_CHIP_IMAGE")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
image = os.path.realpath(os.path.join(incomingPath,name))
transaction.moveFile(image, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
name = transaction.getIncoming().getName()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
#name = transaction.getIncoming().getName()
parents = []
identifier = pattern.findall(name)[0]
if isExpected(identifier):
experiment = identifier[1:5]
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, parentCode))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(parentSampleIdentifier)
# register new experiment and sample
existingExperimentIDs = []
existingExperiments = search_service.listExperiments("/" + space + "/" + project)
numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
for eexp in existingExperiments:
existingExperimentIDs.append(eexp.getExperimentIdentifier())
newExpID = '/' + space + '/' + project + '/' + project + 'E' +str(numberOfExperiments)
while newExpID in existingExperimentIDs:
numberOfExperiments += 1
newExpID = '/' + space + '/' + project + '/' + project + 'E' +str(numberOfExperiments)
newHLATypingExperiment = transaction.createNewExperiment(newExpID, "Q_NGS_HLATYPING")
newHLATypingExperiment.setPropertyValue('Q_CURRENT_STATUS', 'FINISHED')
newHLATypingSample = transaction.createNewSample('/' + space + '/' + 'HLA'+ parentCode, "Q_NGS_HLATYPING")
newHLATypingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
newHLATypingSample.setExperiment(newHLATypingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("Q_NGS_HLATYPING_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(newHLATypingSample)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if key == None:
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName " + name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)
)
foundSamples = ss.searchForSamples(sc)
samplehit = foundSamples[0]
sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier())
parents = samplehit.getParentSampleIdentifiers()
parentcodes = []
for parent in parents:
parentcodes.append(parent.split("/")[-1])
parentInfos = "_".join(parentcodes)
experiment = transaction.getExperimentForUpdate("/" + space + "/" + project + "/" + experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(time.time()).strftime("%Y-%m-%d %H:%M:%S")
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
# Register files
dataSetRes = transaction.createNewDataSet("Q_WF_NGS_RNAEXPRESSIONANALYSIS_RESULTS")
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet("Q_WF_NGS_RNAEXPRESSIONANALYSIS_LOGS")
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
resultsname = incomingPath + "/" + parentInfos + "_workflow_results"
logname = incomingPath + "/" + parentInfos + "_workflow_logs"
os.rename(incomingPath + "/logs", logname)
os.rename(incomingPath + "/result", resultsname)
transaction.moveFile(resultsname, dataSetRes)
transaction.moveFile(logname, dataSetLogs)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName " + name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = ss.searchForSamples(sc)
samplehit = foundSamples[0]
sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier())
parents = samplehit.getParentSampleIdentifiers()
parentcodes = []
for parent in parents:
parentcodes.append(parent.split("/")[-1])
parentInfos = "_".join(parentcodes)
#sample.setPropertyValue("Q_ADDITIONAL_INFO", "Merged from different lanes e.g.")
experiment = transaction.getExperimentForUpdate("/" + space + "/" +
project + "/" +
experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
#Register files
dataSetRes = transaction.createNewDataSet('Q_NGS_RAW_DATA')
dataSetRes.setMeasuredData(False)
dataSetRes.setSample(sample)
resultsname = incomingPath + "/" + parentInfos + "_workflow_results"
#logname = incomingPath+"/"+parentInfos+"_workflow_logs"
#os.rename(incomingPath+"/logs", logname)
os.rename(incomingPath + "/result", resultsname)
transaction.moveFile(resultsname, dataSetRes)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if key == None:
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
# parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, identifier)
)
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
# newVariantCallingExperiment = transaction.createNewExperiment('/' + space + '/' + project + '/' + project + 'E' + str(numberOfExperiments), "Q_NGS_VARIANT_CALLING")
# newVariantCallingSample = transaction.createNewSample('/' + space + '/' + 'VC'+ parentCode, "Q_NGS_VARIANT_CALLING")
# newVariantCallingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
# newVariantCallingSample.setExperiment(newVariantCallingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("FEATUREXML")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
# cegat = False
f = "source_dropbox.txt"
sourceLabFile = open(os.path.join(incomingPath, f))
sourceLab = sourceLabFile.readline().strip()
sourceLabFile.close()
# if sourceLab == 'dmcegat':
# cegat = True
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
for f in os.listdir(incomingPath):
if ".testorig" in f:
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
# elif f.endswith('vcf') and cegat:
# secondaryName = f.split('_')[0]
# entitySample = transaction.getSampleForUpdate('/%s/%s' % (space,parentCode))
# sa.setPropertyValue('Q_SECONDARY_NAME', secondaryName)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
#sample_id = experiment_id+'.'
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName " + name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = ss.searchForSamples(sc)
samplehit = foundSamples[0]
sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier())
experiment = transaction.getExperimentForUpdate("/" + space + "/" +
project + "/" +
experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
#Register files
dataSetRes = transaction.createNewDataSet('Q_NGS_READ_MATCH_ARCHIVE')
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet(
'Q_WF_NGS_16S_TAXONOMIC_PROFILING_LOGS')
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
resultsname = incomingPath + "/" + experiment_id + "_workflow_results"
logname = incomingPath + "/" + experiment_id + "_workflow_logs"
os.rename(incomingPath + "/logs", logname)
os.rename(incomingPath + "/result", resultsname)
transaction.moveFile(resultsname, dataSetRes)
transaction.moveFile(logname, dataSetLogs)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
#parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, identifier))
foundSamples = search_service.searchForSamples(sc)
sampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
#numberOfExperiments = len(search_service.listExperiments("/" + space + "/" + project)) + 1
#newVariantCallingExperiment = transaction.createNewExperiment('/' + space + '/' + project + '/' + project + 'E' + str(numberOfExperiments), "Q_NGS_VARIANT_CALLING")
#newVariantCallingSample = transaction.createNewSample('/' + space + '/' + 'VC'+ parentCode, "Q_NGS_VARIANT_CALLING")
#newVariantCallingSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
#newVariantCallingSample.setExperiment(newVariantCallingExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("IDXML")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
#cegat = False
f = "source_dropbox.txt"
sourceLabFile = open(os.path.join(incomingPath, f))
sourceLab = sourceLabFile.readline().strip()
sourceLabFile.close()
#if sourceLab == 'dmcegat':
#cegat = True
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
for f in os.listdir(incomingPath):
if ".testorig" in f:
os.remove(os.path.realpath(os.path.join(incomingPath, f)))
#elif f.endswith('vcf') and cegat:
#secondaryName = f.split('_')[0]
#entitySample = transaction.getSampleForUpdate('/%s/%s' % (space,parentCode))
#sa.setPropertyValue('Q_SECONDARY_NAME', secondaryName)
transaction.moveFile(incomingPath, dataSet)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, identifier))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(parentSampleIdentifier)
# find or register new experiment
expType = "Q_MS_MEASUREMENT"
msExperiment = None
experiments = search_service.listExperiments("/" + space + "/" + project)
experimentIDs = []
for exp in experiments:
experimentIDs.append(exp.getExperimentIdentifier())
if exp.getExperimentType() == expType:
msExperiment = exp
# no existing experiment for samples of this sample preparation found
if not msExperiment:
expID = experimentIDs[0]
i = 0
while expID in experimentIDs:
i += 1
expNum = len(experiments) + i
expID = '/' + space + '/' + project + '/' + project + 'E' + str(expNum)
msExperiment = transaction.createNewExperiment(expID, expType)
newMSSample = transaction.createNewSample('/' + space + '/' + 'MS'+ parentCode, "Q_MS_RUN")
newMSSample.setParentSampleIdentifiers([sa.getSampleIdentifier()])
newMSSample.setExperiment(msExperiment)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MS_MZML_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(newMSSample)
transaction.moveFile(incomingPath, dataSet)
def sampleExists(searchService, sampleCode):
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = searchService.searchForSamples(sc)
if len(foundSamples) > 0:
return True
return False
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.PROJECT, project));
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
foundSamples = search_service.searchForSamples(sc)
foundSamplesFilter = [s for s in foundSamples if 'ENTITY' not in s.getCode()]
offset = 0
exists = True
while exists:
# create new barcode
newBarcode = getNextFreeBarcode(project, len(foundSamplesFilter) + len(newTestSamples) + offset)
# check if barcode already exists in database
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, newBarcode))
found = search_service.searchForSamples(pc)
if len(found) == 0:
exists = False
else:
offset += 1
return newBarcode
def get_dataset_for_permid(transaction, permid):
search_service = transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, permid))
found = list(search_service.searchForDataSets(criteria))
if len(found) == 1:
return found[0]
else:
return None
def runs(self):
"""Return *all* runs in the db.
TODO this is madness!! At least, this should only return
runs in this project.
"""
search = self.transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, 'Q_MS_RUN'))
return search.searchForSamples(criteria)
def get_dataset_for_name(transaction, dataset_name):
search_service = transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
MatchClause.createPropertyMatch('NAME', dataset_name))
found = list(search_service.searchForDataSets(criteria))
if len(found) == 1:
print("DataSetCode of found dataset = " + found[0].getDataSetCode())
return transaction.getDataSetForUpdate(found[0].getDataSetCode())
#return found[0]
else:
return None
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(PARENT)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code)
)
sample = searchService.searchForSamples(sc)[0]
handleSample(sample, tableBuilder)
def _getExperimentSample(collectionPermId, expSamplePermId):
"""Retrieve the experiment sample."""
# Get the experiment sample
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
expSamplePermId)
)
expCriteria = SearchCriteria()
expCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
collectionPermId)
)
# Add the experiment subcriteria
sampleCriteria.addSubCriteria(
SearchSubCriteria.createExperimentCriteria(
expCriteria)
)
# Search
expSampleList = searchService.searchForSamples(sampleCriteria)
if len(expSampleList) != 1:
return None
# Return the experiment sample
return expSampleList[0]
def _getDataSetsForSample(sampleIdentifier, dataSetType):
"""Return the dataSet of given type for specified sample."""
# Set search criteria to retrieve the dataSet associated with the sample
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
dataSetType)
)
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.CODE,
sampleIdentifier)
)
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(
sampleCriteria)
)
dataSetList = searchService.searchForDataSets(searchCriteria)
if len(dataSetList) != 1:
[]
# Return the dataSet
return dataSetList
def listSamplesForExperiment(searchService, sampleType, expID):
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, sampleType))
ec = SearchCriteria()
ec.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, expID))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(ec))
existingSamples = searchService.searchForSamples(sc)
return existingSamples
def _getDataSetForTube(self, tubeCode=None):
"""
Get the datasets belonging to the tube with specified tube code.
If none is found, return [].
If no tubeCode is given, it is assumed that the tube is the passed
entity with code self._entityCode.
"""
if tubeCode is None:
tubeCode = self._entityCode
# Set search criteria to retrieve the dataset contained in the tube
searchCriteria = SearchCriteria()
tubeCriteria = SearchCriteria()
tubeCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, tubeCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(tubeCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for tube " \
"with code " + tubeCode + "."
self._logger.error(self._message)
# Return
return dataSets
def _getDataSetForTube(self, tubeCode):
"""
Get the datasets belonging to the tube with specified tube code.
If none is found, return [].
"""
if _DEBUG:
self._logger.info("Searching for tube with code " + tubeCode)
# Set search criteria to retrieve the dataset contained in the tube
searchCriteria = SearchCriteria()
tubeCriteria = SearchCriteria()
tubeCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE,
tubeCode))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(tubeCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if _DEBUG:
self._logger.info("Retrieved " + str(len(dataSets)) + \
" datasets for tube with code " + tubeCode)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for tube " \
"with code " + tubeCode + "."
self._logger.error(self._message)
# Return
return dataSets
def _getDataSetForWell(self, wellId=None):
"""
Get the datasets belonging to the well with specified code. If none
are found, return [].
If no wellId is given, it is assumed that the well is the passed
entity with code self._entityId.
"""
if wellId is None:
wellId = self._entityId
# Set search criteria to retrieve the dataset contained in the well
searchCriteria = SearchCriteria()
wellCriteria = SearchCriteria()
wellCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.CODE,
wellId))
searchCriteria.addSubCriteria(
SearchSubCriteria.createSampleCriteria(wellCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for well " \
"with code " + wellId + "."
self._logger.error(self._message)
# Return
return dataSets
def isCurrentMSRun(tr, parentExpID, msExpID):
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.TYPE, "Q_MS_RUN")
)
foundSamples = search_service.searchForSamples(sc)
for samp in foundSamples:
currentMSExp = samp.getExperiment()
if currentMSExp.getExperimentIdentifier() == msExpID:
for parID in samp.getParentSampleIdentifiers():
parExp = tr.getSampleForUpdate(parID).getExperiment().getExperimentIdentifier()
if parExp == parentExpID:
return True
return False
def runs(self):
"""Return *all* runs in the db.
TODO this is madness!! At least, this should only return
runs in this project.
"""
search = self.transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE,
'Q_MS_RUN'
)
)
return search.searchForSamples(criteria)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
nameSplit = name.split("-")
space = nameSplit[0]
project = pPattern.findall(nameSplit[1])[0]
experiment_id = ePattern.findall(nameSplit[2])[0]
sampleCode = nameSplit[-1]
if not experiment_id:
print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name
ss = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = ss.searchForSamples(sc)
sample = foundSamples[0]
sample = transaction.getSampleForUpdate(sample.getSampleIdentifier())
experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id)
experiment.setPropertyValue("Q_WF_STATUS", "FINISHED")
endpoint = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
experiment.setPropertyValue("Q_WF_FINISHED_AT", endpoint)
sample.setExperiment(experiment)
#Register files
dataSetRes = transaction.createNewDataSet('Q_WF_EDDA_BENCHMARK_RESULTS')
dataSetRes.setMeasuredData(False)
dataSetLogs = transaction.createNewDataSet('Q_WF_EDDA_BENCHMARK_LOGS')
dataSetLogs.setMeasuredData(False)
dataSetRes.setSample(sample)
dataSetLogs.setSample(sample)
transaction.moveFile(incomingPath+"/result", dataSetRes)
transaction.moveFile(incomingPath+"/logs", dataSetLogs)
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
space = "DEFAULT" #will be set later
project = "DEFAULT"
experiment = "DEFAULT"
sample_id = "DEFAULT"
extension = name.split(".")[-1]
identifier = pattern.findall(name)[0]
if isExpected(identifier, extension):
print "found identifier"
#experiment = identifier[1:5]
project = identifier[4:9]
sample_id = identifier[:14]
else:
print "The identifier "+identifier+" did not match the pattern MSQCQ[A-Z]{4}\d{3}\w{2}, the checksum was wrong or the file is not of type ."+ext
#Register file
dataSet = transaction.createNewDataSet('Q_WF_MS_QUALITYCONTROL_RESULTS')
dataSet.setMeasuredData(False)
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sample_id))
foundSamples = search_service.searchForSamples(sc)
if foundSamples.size() > 0:
sa = transaction.getSampleForUpdate(foundSamples[0].getSampleIdentifier())
transaction.moveFile(incomingPath, dataSet)
dataSet.setSample(sa)
else:
#Sample not found, something went wrong and the file is attached to an "Unknown" sample
transaction.moveFile(incomingPath, dataSet)
newSample = transaction.createNewSample('/' + space + '/' + sample_id, 'UNKNOWN')
newSample.setExperiment(experiment)
dataSet.setSample(newSample)
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
offset = 0
exists = True
while exists:
n = str(len(newTestSamples)+1+offset) #for future programmers: sorry if this reaches > 999 !
code = project+n.zfill(3)+"X" # should go to other letters in that case
code = code+checksum.checksum(code)
pc = SearchCriteria()
pc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
found = search_service.searchForSamples(pc)
print found
if len(found) == 0:
exists = False
else:
offset += 1
return code
def isCurrentMSRun(tr, parentExpID, msExpID):
"""Ask Andreas"""
search_service = tr.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.TYPE, "Q_MS_RUN"))
foundSamples = search_service.searchForSamples(sc)
for samp in foundSamples:
currentMSExp = samp.getExperiment()
if currentMSExp.getExperimentIdentifier() == msExpID:
for parID in samp.getParentSampleIdentifiers():
parExp = (tr.getSampleForUpdate(
parID).getExperiment().getExperimentIdentifier())
if parExp == parentExpID:
return True
return False
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
code = pattern.findall(name)[0]
#such checks should not be needed here as it's not raw data
#if isExpected(code):
# project = code[2:7]
# #parentCode = code[2:]
#else:
# print "The identifier "+code+" did not match the pattern MSQ[A-Z]{4}\d{3}\w{2}"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, "MS"+code))
# wait for registration of ms sample to finish (done in raw dropbox after mzml marker is created)
timeout = 10
while timeout > 0:
timeout = timeout - 1
time.sleep(5)
foundSamples = search_service.searchForSamples(sc)
if foundSamples > 0:
break
sampleIdentifier = foundSamples[0].getSampleIdentifier()
#space = foundSamples[0].getSpace()
sa = transaction.getSampleForUpdate(sampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_MS_RAW_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(sa)
transaction.moveFile(incomingPath, dataSet)
def process(tr, parameters, tableBuilder):
ids = sorted(parameters.get("ids"))
types = parameters.get(
"types") #sample types (tiers) that are requested for the tsv
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
sampleMap = {}
voc = searchService.getVocabulary("Q_NCBI_TAXONOMY")
for id in ids:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE,
id.split('/')[-1]))
sample = searchService.searchForSamples(sc)[0]
sampleMap[id] = sample
for id in ids:
sample = sampleMap[id]
if sample.getSampleType() in types:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME,
sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource(id, sampleMap, voc.getTerms()))
row.setCell(EXTERNAL_ID,
sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType == "CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def getsample(qcode, transaction):
"""Resturns a immutable sample object for a given
QBIC barcode."""
sserv = transaction.getSearchService()
scrit = SearchCriteria()
scrit.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, qcode))
result = sserv.searchForSamples(scrit)
if not result:
raise mtbutils.MTBdropboxerror(
'No matching sample found in openBIS for code {}.'.format(qcode))
if len(result) > 1:
raise mtbutils.MTBdropboxerror(
'More than one sample found in openBIS for code {}.'.format(qcode))
return result[0]
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(XML)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
sample = searchService.searchForSamples(sc)[0]
row = tableBuilder.addRow()
row.setCell(CODE, sample.getCode())
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
def get_space_from_project(transaction, project):
search_service = transaction.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.PROJECT, project))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
foundSamples = search_service.searchForSamples(sc)
space = foundSamples[0].getSpace()
return space
def _getAllPlates(self):
"""
Get all plates in the experiment. Returns [] if none are found.
"""
# Set search criteria to retrieve all plates in the experiment
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE, self._experimentPrefix + "_PLATE"))
expCriteria = SearchCriteria()
expCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID, self._experiment.permId))
searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(expCriteria))
plates = searchService.searchForSamples(searchCriteria)
if len(plates) == 0:
self._message = "Could not retrieve plates for experiment with code " + self._experimentCode + "."
return plates
# Return the plates
return plates
def _getSamples(self, expSampleType, expSamplePermId, sampleType):
"""
Return a list of datasets of requested type belonging to the MICROSCOPY_EXPERIMENT sample
and a specific sample of type MICROSCOPY_SAMPLE_TYPE.
If none are found, return [].
"""
if _DEBUG:
self._logger.info("* Requested experiment sample type: " + expSampleType)
self._logger.info("* Requested experiment sample permId: " + expSamplePermId)
self._logger.info("* Requested sample type: " + sampleType)
# Search samples of type MICROSCOPY_SAMPLE_TYPE
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
sampleType)
)
# Search parent sample of type MICROSCOPY_EXPERIMENT with specified permId
sampleParentCriteria = SearchCriteria()
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
expSampleType))
sampleParentCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
expSamplePermId))
# Add the parent sample subcriteria
sampleCriteria.addSubCriteria(
SearchSubCriteria.createSampleParentCriteria(
sampleParentCriteria
)
)
# Search
samples = searchService.searchForSamples(sampleCriteria)
# Return
return samples
def process(tr, parameters, tableBuilder):
#ids = sorted(parameters.get("ids"))
types = parameters.get(
"types") #sample types (tiers) that are requested for the tsv
project = parameters.get("project")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
#search all samples of project
search = tr.getSearchService()
sc = SearchCriteria()
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.PROJECT, project))
sc.addSubCriteria(SearchSubCriteria.createExperimentCriteria(pc))
fetchOptions = EnumSet.of(SampleFetchOption.ANCESTORS,
SampleFetchOption.PROPERTIES)
allSamples = search.searchForSamples(sc, fetchOptions)
#filter all samples by types
samples = []
for s in allSamples:
if s.getSampleType() in types:
samples.append(s)
#sort remaining samples-
samples = sorted(samples)
voc = search.getVocabulary("Q_NCBI_TAXONOMY")
for s in samples:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME,
sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource([sample], voc.getTerms(), []))
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType == "CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def _bio_sample(self, barcode):
"""Find a sample in openbis by it's barcode.
Since we use the barcode an unique identifier, there should never
be more than one sample with a given barcode.
"""
search = self._transaction.getSearchService()
criteria = SearchCriteria()
barcode_match = SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, barcode)
criteria.addMatchClause(barcode_match)
samples = search.searchForSamples(criteria)
if len(samples) > 1:
raise RuntimeError("Found more than one sample for barcode %s." %
barcode)
if not samples:
raise ValueError("Could not find a sample for barcode %s" %
barcode)
sample = samples[0]
return sample.getSpace(), self.barcode[:5], sample
def aggregate(parameters, tableBuilder):
codes = parameters.get("codes")
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(XML)
for code in codes:
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code)
)
sample = searchService.searchForSamples(sc)[0]
row = tableBuilder.addRow()
row.setCell(CODE, sample.getCode())
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
def createNewBarcode(project, tr):
search_service = tr.getSearchService()
offset = 0
exists = True
while exists:
n = str(len(newTestSamples) + 1 +
offset) #for future programmers: sorry if this reaches > 999 !
code = project + n.zfill(
3) + "X" # should go to other letters in that case
code = code + checksum.checksum(code)
pc = SearchCriteria()
pc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, code))
found = search_service.searchForSamples(pc)
print found
if len(found) == 0:
exists = False
else:
offset += 1
return code
def _getDataSetForWell(self, wellCode=None):
"""
Get the datasets belonging to the well with specified code. If none
are found, return [].
If no wellCode is given, it is assumed that the well is the passed
entity with code self._entityCode.
"""
if wellCode is None:
wellCode = self._entityCode
# Set search criteria to retrieve the dataset contained in the well
searchCriteria = SearchCriteria()
wellCriteria = SearchCriteria()
wellCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, wellCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleCriteria(wellCriteria))
dataSets = searchService.searchForDataSets(searchCriteria)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for well " \
"with code " + wellCode + "."
# Return
return dataSets
def _retrieveSampleWithTypeAndPermId(self, samplePermId, sampleType):
"""
Retrieve a sample belonging to current experiment
sample and collection having specified type and perm id.
"""
# The sample is of type 'sampleType' and has id 'sampleId'
searchCriteria = SearchCriteria()
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
sampleType))
searchCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
samplePermId))
# Now search
samples = searchService.searchForSamples(searchCriteria)
if len(samples) != 1:
self._message = "Sample with id " + sampleId + \
" and type " + sampleType + "not found!"
self._logger.error(self._message)
return None
# Return the sample
return samples[0]
def _getFlowExperimentSample(self):
"""Find the {FLOW}_EXPERIMENT sample with given Id."""
# Inform
if _DEBUG:
self._logger.info("Retrieving experiment sample of code " + \
self._expSampleId + ", permId " + self._expSamplePermId + \
" and type " + self._expSampleType)
# Search sample of type MICROSCOPY_EXPERIMENT with specified CODE
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.TYPE,
self._expSampleType))
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
# Search
samples = searchService.searchForSamples(sampleCriteria)
if len(samples) == 0:
samples = []
self._message = "Could not retrieve " + self._expSampleType + " sample with code " + \
self._expSampleId + ", permId " + self._expSamplePermId + \
" and type " + self._expSampleType + "."
self._logger.error(self._message)
return samples
if _DEBUG:
self._logger.info("Successfully returned sample " +
self._expSampleId)
# Return
return samples[0]
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
found = pattern.findall(name)
if len(found) > 0:
code = found[0]
if not isExpected(code):
print "The code " + code + " did not match the checksum"
else:
code = alt_pattern.findall(name)[0]
project = code[:5]
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, code))
foundSamples = search_service.searchForSamples(sc)
sampleID = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
sample = transaction.getSampleForUpdate(sampleID)
# create new dataset
dataSet = transaction.createNewDataSet("Q_DOCUMENT")
dataSet.setMeasuredData(False)
dataSet.setSample(sample)
transaction.moveFile(incomingPath, dataSet)
def process(tr, parameters, tableBuilder):
ids = sorted(parameters.get("ids"))
types = parameters.get("types") #sample types (tiers) that are requested for the tsv
tableBuilder.addHeader(CODE)
tableBuilder.addHeader(SECONDARY_NAME)
tableBuilder.addHeader(SOURCE)
tableBuilder.addHeader(EXTERNAL_ID)
tableBuilder.addHeader(SAMPLE_TYPE)
tableBuilder.addHeader(XML)
tableBuilder.addHeader(TIER)
sampleMap = {}
voc = searchService.getVocabulary("Q_NCBI_TAXONOMY")
for id in ids:
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, id.split('/')[-1]))
sample = searchService.searchForSamples(sc)[0]
sampleMap[id] = sample
for id in ids:
sample = sampleMap[id]
if sample.getSampleType() in types:
code = sample.getCode()
row = tableBuilder.addRow()
row.setCell(CODE, code)
row.setCell(SECONDARY_NAME, sample.getPropertyValue("Q_SECONDARY_NAME"))
row.setCell(SOURCE, fetchSource(id, sampleMap, voc.getTerms()))
row.setCell(EXTERNAL_ID, sample.getPropertyValue("Q_EXTERNALDB_ID"))
extrType = sample.getPropertyValue("Q_PRIMARY_TISSUE")
if not extrType:
extrType = sample.getPropertyValue("Q_SAMPLE_TYPE")
if not extrType:
extrType = ""
if extrType=="CELL_LINE":
extrType = sample.getPropertyValue("Q_TISSUE_DETAILED")
row.setCell(SAMPLE_TYPE, extrType)
row.setCell(XML, sample.getPropertyValue("Q_PROPERTIES"))
row.setCell(TIER, sample.getSampleType())
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
experiment = identifier[1:5]
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier " + identifier + " did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(
SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, parentCode))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
parentSample = transaction.getSampleForUpdate(parentSampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_PEPTIDE_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(parentSample)
transaction.moveFile(incomingPath, dataSet)
def _bio_sample(self, barcode):
"""Find a sample in openbis by it's barcode.
Since we use the barcode an unique identifier, there should never
be more than one sample with a given barcode.
"""
search = self._transaction.getSearchService()
criteria = SearchCriteria()
barcode_match = SearchCriteria.MatchClause.createAttributeMatch(
SearchCriteria.MatchClauseAttribute.CODE, barcode
)
criteria.addMatchClause(barcode_match)
samples = search.searchForSamples(criteria)
if len(samples) > 1:
raise RuntimeError(
"Found more than one sample for barcode %s." % barcode
)
if not samples:
raise ValueError(
"Could not find a sample for barcode %s" % barcode
)
sample = samples[0]
return sample.getSpace(), self.barcode[:5], sample
def _getMicroscopyExperimentSample(self):
"""Find the MICROSCOPY_EXPERIMENT sample with given Id."""
if _DEBUG:
self._logger.info("Retrieving sample with permId " +
self._expSamplePermId + " and type " +
self._expSampleType + " from experiment " +
"with permId " + self._collection.permId +
" and type " + self._collection.getExperimentType())
# Search sample of type MICROSCOPY_EXPERIMENT with specified permId
sampleCriteria = SearchCriteria()
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.TYPE,
self._expSampleType))
sampleCriteria.addMatchClause(
MatchClause.createAttributeMatch(
MatchClauseAttribute.PERM_ID,
self._expSamplePermId))
# Search
samples = searchService.searchForSamples(sampleCriteria)
if len(samples) == 0:
samples = []
self._message = "Could not retrieve " + self._expSampleType + \
" sample with permId " + self._expSamplePermId + \
" from experiment with permId " + self._collection.permId + "."
self._logger.error(self._message)
return samples
if _DEBUG:
self._logger.info("Successfully returned sample with permId " + self._expSamplePermId)
# Return
return samples[0]
def process(transaction):
context = transaction.getRegistrationContext().getPersistentMap()
# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()
key = context.get("RETRY_COUNT")
if (key == None):
key = 1
# Get the name of the incoming file
name = transaction.getIncoming().getName()
identifier = pattern.findall(name)[0]
if isExpected(identifier):
experiment = identifier[1:5]
project = identifier[:5]
parentCode = identifier[:10]
else:
print "The identifier "+identifier+" did not match the pattern Q[A-Z]{4}\d{3}\w{2} or checksum"
search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, parentCode))
foundSamples = search_service.searchForSamples(sc)
parentSampleIdentifier = foundSamples[0].getSampleIdentifier()
space = foundSamples[0].getSpace()
parentSample = transaction.getSampleForUpdate(parentSampleIdentifier)
# create new dataset
dataSet = transaction.createNewDataSet("Q_PEPTIDE_DATA")
dataSet.setMeasuredData(False)
dataSet.setSample(parentSample)
transaction.moveFile(incomingPath, dataSet)
def _getDataSetsForPlate(self, plateCode=None):
"""
Return a list of datasets belonging to the plate with specified ID
optionally filtered by self._specimen. If none are found, return [].
If no plateCode is given, it is assumed that the plate is the passed
entity with code self._entityCode.
"""
if plateCode is None:
plateCode = self._entityCode
# Set search criteria to retrieve all wells contained in the plate
searchCriteria = SearchCriteria()
plateCriteria = SearchCriteria()
plateCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, plateCode))
searchCriteria.addSubCriteria(SearchSubCriteria.createSampleContainerCriteria(plateCriteria))
wells = searchService.searchForSamples(searchCriteria)
if len(wells) == 0:
self._message = "Could not retrieve wells for plate with " \
"code " + plateCode + "."
self._logger.error(self._message)
return wells
# Check that the specimen matches (if needed)
if self._specimen != "":
wells = [well for well in wells if \
well.getPropertyValue(self._experimentPrefix + "_SPECIMEN") == self._specimen]
# Now iterate over the samples and retrieve their datasets
dataSets = []
for well in wells:
wellCode = well.getCode()
dataSetsForWell = self._getDataSetForWell(wellCode)
dataSets.extend(dataSetsForWell)
if len(dataSets) == 0:
self._message = "Could not retrieve datasets for wells in " \
"plate with code " + plateCode + " from experiment " \
"with code " + self._experimentCode + "."
self._logger.error(self._message)
# Return
return dataSets