class ProtPKPDExponentialFit(ProtPKPDFitBase):
""" Fit a set of exponentials. The observed measurement is modelled as Y=sum_{i=1}^N c_i exp(-lambda_i * X).\n
Confidence intervals calculated by this fitting may be pessimistic because it assumes that all model parameters
are independent, which are not. Use Bootstrap estimates instead.\n
Protocol created by http://www.kinestatpharma.com\n"""
_label = 'fit exponentials'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form, fullForm=True):
self._defineParams1(form,"t","Cp")
if fullForm:
form.addParam('fitType', params.EnumParam, choices=["Linear","Logarithmic","Relative"], label="Fit mode", default=1,
help='Linear: sum (Cobserved-Cpredicted)^2\nLogarithmic: sum(log10(Cobserved)-log10(Cpredicted))^2\n'\
"Relative: sum ((Cobserved-Cpredicted)/Cobserved)^2")
form.addParam('Nexp', params.IntParam, label="Number of exponentials", default=1,
help='Number of exponentials to fit')
else:
self.fitType=Integer()
self.fitType.set(1)
self.Nexp=Integer()
self.Nexp.set(1)
form.addParam('bounds', params.StringParam, label="Amplitude and time constant bounds", default="", expertLevel=LEVEL_ADVANCED,
help='Bounds for the c_i amplitudes and lambdas.\nExample 1: (0,10);(0,1e-2) -> c1 in (0,10), lambda1 in (0,1e-2)\n'\
'Example 2: (0,10);(0,1e-2);(0,1);(0,1e-1) -> c1 in (0,10), lambda1 in (0,1e-2), c2 in (0,1), lambda2 in (0,1e-1)')
form.addParam('confidenceInterval', params.FloatParam, label="Confidence interval=", default=95, expertLevel=LEVEL_ADVANCED,
help='Confidence interval for the fitted parameters')
if fullForm:
form.addParam('reportX', params.StringParam, label="Evaluate at X=", default="", expertLevel=LEVEL_ADVANCED,
help='Evaluate the model at these X values\nExample 1: [0,5,10,20,40,100]\nExample 2: 0:0.55:10, from 0 to 10 in steps of 0.5')
else:
self.reportX=String()
self.reportX.set("")
def getListOfFormDependencies(self):
return [self.predictor.get(), self.predicted.get(), self.fitType.get(), self.bounds.get()]
def createModel(self):
return PKPDExponentialModel()
def setupFromFormParameters(self):
self.model.Nexp=self.Nexp.get()
#--------------------------- INFO functions --------------------------------------------
def _warnings(self):
warnings = []
experiment = self.readExperiment(self.getInputExperiment().fnPKPD,show=False)
incorrectList = experiment.getNonBolusDoses()
if len(incorrectList)>0:
warnings.append("This protocol is meant only for intravenous bolus regimens. Check the doses for %s"%(','.join(incorrectList)))
return warnings
def _validate(self):
errors=ProtPKPDFitBase._validate(self)
if self.Nexp.get()<1:
errors.append("The number of exponentials has to be larger than 0")
return errors
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
INPUT_HELP = """ Input volume(s) for ResMap.
Required volume properties:
1. The particle must be centered in the volume.
2. The background must not been masked out.
Desired volume properties:
1. The volume has not been filtered in any way (low-pass filtering, etc.)
2. The volume has a realistic noise spectrum.
This is sometimes obtained by so-called amplitude correction.
While a similar effect is often obtained by B-factor sharpening,
please make sure that the spectrum does not blow up near Nyquist.
"""
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
self.plotData = String() # store some values for later plot
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('useSplitVolume', params.BooleanParam, default=False,
label="Use half volumes?",
help='Use to split volumes for gold-standard FSC.')
form.addParam('inputVolume', params.PointerParam,
pointerClass='Volume', condition="not useSplitVolume",
label="Input volume", important=True,
help=self.INPUT_HELP)
form.addParam('volumeHalf1', params.PointerParam,
label="Volume half 1", important=True,
pointerClass='Volume', condition="useSplitVolume",
help=self.INPUT_HELP)
form.addParam('volumeHalf2', params.PointerParam,
pointerClass='Volume', condition="useSplitVolume",
label="Volume half 2", important=True,
help=self.INPUT_HELP)
form.addParam('applyMask', params.BooleanParam, default=False,
label="Mask input volume?",
help="It is not necessary to provide ResMap with a mask "
"volume. The algorithm will attempt to estimate a "
"mask volume by low-pass filtering the input volume "
"and thresholding it using a heuristic procedure.\n"
"If the automated procedure does not work well for "
"your particle, you may provide a mask volume that "
"matches the input volume in size and format. "
"The mask volume should be a binary volume with zero "
"(0) denoting the background/solvent and some positive"
"value (0+) enveloping the particle.")
form.addParam('maskVolume', params.PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
form.addParam('whiteningLabel', params.LabelParam, important=True,
label="It is strongly recommended to use the "
"pre-whitening wizard.")
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', params.FloatParam, default=10,
label="Angstroms")
line.addParam('prewhitenRamp', params.FloatParam, default=1,
label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', params.FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help="Default (0): algorithm will start a just above\n"
" 2*voxelSize until 4*voxelSize. \n"
"These fields are provided to accelerate computation "
"if you are only interested in analyzing a specific "
"resolution range. It is usually a good idea to provide "
"a maximum resolution value to save time. Another way to "
"save computation is to provide a larger step size.")
line.addParam('minRes', params.FloatParam, default=0, label='Min')
line.addParam('maxRes', params.FloatParam, default=0, label='Max')
group.addParam('pVal', params.FloatParam, default=0.05,
label='Confidence level:',
help="P-value, usually between [0.01, 0.05].\n\n"
"This is the p-value of the statistical hypothesis test "
"on which ResMap is based on. It is customarily set to "
"0.05 although you are welcome to reduce it (e.g. 0.01) "
"if you would like to obtain a more conservative result. "
"Empirically, ResMap results are not much affected by the p-value.")
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
if self.useSplitVolume:
inputs = [self.volumeHalf1, self.volumeHalf2]
self.inputVolume.set(None)
else:
inputs = [self.inputVolume]
self.volumeHalf1.set(None)
self.volumeHalf2.set(None)
locations = [i.get().getLocation() for i in inputs]
self._insertFunctionStep('convertInputStep', *locations)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropriate parameters. """
results = self.runResmap(self._getPath())
self.histogramData.set(dumps(results['resHisto']))
plotDict = {'minRes': results['minRes'],
'maxRes': results['maxRes'],
'orig_n': results['orig_n'],
'n': results['n'],
'currentRes': results['currentRes']
}
self.plotData.set(dumps(plotDict))
self._store(self.histogramData, self.plotData)
self.savePlots(results)
def savePlots(self, results=None):
""" Store png images of the plots to be used as images, """
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
self._plotVolumeSlices().savefig(self._getExtraPath('volume1.map.png'))
plot = self._plotResMapSlices(results['resTOTALma'])
plot.savefig(self._getExtraPath('volume1_resmap.map.png'))
self._plotHistogram().savefig(self._getExtraPath('histogram.png'))
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
if self.useSplitVolume:
half1 = self.volumeHalf1.get()
half2 = self.volumeHalf2.get()
if half1.getSamplingRate() != half2.getSamplingRate():
errors.append('The selected half volumes have not the same pixel size.')
if half1.getXDim() != half2.getXDim():
errors.append('The selected half volumes have not the same dimensions.')
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'vxSize': self.volumeHalf1.get().getSamplingRate(),
'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'vxSize': self.inputVolume.get().getSamplingRate(),
'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self, **kwargs):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'), **kwargs)
return Plotter(figure=fig)
def _plotResMapSlices(self, data=None, **kwargs):
from ResMap_visualization import plotResMapVolume
plotDict = loads(self.plotData.get())
if data is None:
data = self._getVolumeMatrix('volume1_resmap.map')
data = np.ma.masked_where(data > plotDict['currentRes'], data, copy=True)
kwargs.update(plotDict)
fig = plotResMapVolume(data, **kwargs)
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
class EmpiarDepositor(EMProtocol):
"""
Deposit image sets to empiar
"""
_label = 'Empiar deposition'
_ih = ImageHandler()
_imageSetCategories = {
"SetOfMicgrographs":
"T1",
"SetOfMovies":
'T2',
'T3':
'micrographs - focal pairs - unprocessed',
'T4':
'micrographs - focal pairs - contrast inverted',
'T5':
'picked particles - single frame - unprocessed',
'T6':
'picked particles - multiframe - unprocessed',
'T7':
'picked particles - single frame - processed',
'T8':
'picked particles - multiframe - processed',
'T9':
'tilt series',
'T10':
'class averages',
'OT':
'other, in this case please specify the category in the second element.'
}
_imageSetFormats = {
'mrc': 'T1',
'mrcs': 'T2',
'tiff': 'T3',
'img': 'T4', # imagic
'dm3': 'T5',
'dm4': 'T6',
'spi': 'T7', # spider
}
_experimentTypes = ['1', '2', '3', '4', '5', '6']
_releaseDateTypes = ["IM", "EP", "HP", "H1"]
_countryCodes = [
'AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT',
'AU', 'AW', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ',
'BL', 'BM', 'BN', 'BO', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA',
'CC', 'CD', 'CF', 'CG', 'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR',
'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', 'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ',
'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI', 'FJ', 'FK', 'FM', 'FO',
'FR', 'FX', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL', 'GM',
'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN',
'HR', 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS',
'IT', 'JE', 'JM', 'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP',
'KR', 'KW', 'KY', 'KZ', 'LA', 'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT',
'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME', 'MF', 'MG', 'MH', 'MK', 'ML',
'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', 'MV', 'MW', 'MX',
'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP', 'NR',
'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN',
'PR', 'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA',
'SB', 'SC', 'SD', 'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN',
'SO', 'SR', 'SS', 'ST', 'SV', 'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG',
'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', 'TR', 'TT', 'TV', 'TW', 'TZ',
'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE', 'VG', 'VI', 'VN',
'VU', 'WF', 'WS', 'XK', 'YE', 'YT', 'ZA', 'ZM', 'ZW'
]
_voxelTypes = {
_ih.DT_UCHAR: 'T1', # 'UNSIGNED BYTE'
_ih.DT_SCHAR: 'T2', # 'SIGNED BYTE'
_ih.DT_USHORT: 'T3', # 'UNSIGNED 16 BIT INTEGER'
_ih.DT_SHORT: 'T4', # 'SIGNED 16 BIT INTEGER'
_ih.DT_UINT: 'T5', # 'UNSIGNED 32 BIT INTEGER'
_ih.DT_INT: 'T6', # 'SIGNED 32 BIT INTEGER'
_ih.DT_FLOAT: 'T7' # '32 BIT FLOAT'
}
DEPOSITION_SCHEMA = 'empiar_deposition.schema.json'
DEPOSITION_TEMPLATE = 'empiar_deposition_template.json'
OUTPUT_DEPO_JSON = 'deposition.json'
OUTPUT_WORKFLOW = 'workflow.json'
IMGSET_KEY = 'imagesets'
IMGSET_NAME = "name"
IMGSET_DIR = "directory"
IMGSET_CAT = "category"
IMGSET_HEADER_FORMAT = "header_format"
IMGSET_DATA_FORMAT = "data_format"
IMGSET_SIZE = "num_images_or_tilt_series"
IMGSET_FRAMES = "frames_per_image"
IMGSET_FRAME_MIN = "frame_range_min"
IMGSET_FRAME_MAX = "frame_range_max"
IMGSET_VOXEL_TYPE = "voxel_type"
IMGSET_PIXEL_WIDTH = "pixel_width"
IMGSET_PIXEL_HEIGHT = "pixel_height"
IMGSET_DETAILS = "details"
IMGSET_WIDTH = "image_width"
IMGSET_HEIGHT = "image_height"
_imageSetTemplate = {
IMGSET_NAME: "",
IMGSET_DIR: "/data/%s",
IMGSET_CAT: "('%s', '%s')",
IMGSET_HEADER_FORMAT: "('%s', '%s')",
IMGSET_DATA_FORMAT: "('%s', '%s')",
IMGSET_SIZE: 0,
IMGSET_FRAMES: 0,
IMGSET_FRAME_MIN: None,
IMGSET_FRAME_MAX: None,
IMGSET_VOXEL_TYPE: "('%s', '%s')",
IMGSET_PIXEL_WIDTH: None,
IMGSET_PIXEL_HEIGHT: None,
IMGSET_DETAILS: "",
IMGSET_WIDTH: 0,
IMGSET_HEIGHT: 0
}
def __init__(self, **kwargs):
EMProtocol.__init__(self, **kwargs)
self.workflowDicts = []
self.entryAuthorStr = ""
self.workflowPath = String()
self.depositionJsonPath = String()
#--------------- DEFINE param functions ---------------
def _defineParams(self, form):
form.addSection(label='Entry')
# form.addParam('workflowJson', params.PathParam,
# label='Workflow json', allowsNull=True,
# help='Path to the workflow json (obtained using the export option (right click on'
# 'one of your selected protocols). Will generate json of all protocols if not provided.')
form.addParam(
"submit",
params.BooleanParam,
label="Submit deposition",
default=True,
help="Set to false to avoid submitting the deposition to empiar "
"(it will just be created locally).")
form.addParam("resume",
params.BooleanParam,
label="Resume upload",
default=False,
condition='submit',
help="Is this a continuation of a previous upload?")
form.addParam(
'entryID',
params.StringParam,
label="Entry ID",
condition="resume",
important=True,
help="EMPIAR entry ID - use if you wanna resume an upload")
form.addParam('uniqueDir',
params.StringParam,
important=True,
label="Unique directory",
condition="resume",
help="EMPIAR directory assigned to this deposition ID")
form.addParam(
'depositionJson',
params.PathParam,
important=True,
label="Deposition json",
condition="resume",
help=
"Path to the json file of the deposition we're about to resume.")
form.addParam(
'jsonTemplate',
params.PathParam,
condition='not resume',
label="Custom json (Optional)",
allowsNull=True,
help=
"Path to a customized template of the EMPIAR submission json, if you don't want to use the "
"default one.")
form.addParam(
'entryTopLevel',
params.StringParam,
label="Top level folder",
validators=[params.NonEmpty],
important=True,
help=
"How you want to name the top level folder of the empiar entry. \n This should be a "
"simple and descriptive name without special characters (:,?, spaces, etc). \n"
"If you're resuming an upload, this should be the same name you used to create the folder."
)
form.addParam(
'entryTitle',
params.StringParam,
label="Entry title",
important=True,
condition="not resume",
help=
"EMPIAR entry title. This should not be empty if not using a custom template."
)
form.addParam(
'entryAuthor',
params.StringParam,
label="Entry author",
important=True,
condition="not resume",
help=
'EMPIAR entry author in the form "LastName, Initials" e.g. Smith, JW\n'
'This should not be empty if not using a custom template.')
form.addParam(
'experimentType',
params.EnumParam,
label="Experiment type",
condition="not resume",
choices=self._experimentTypes,
default=2,
important=True,
help="EMPIAR experiment type:\n"
"1 - image data collected using soft x-ray tomography\n"
"2 - simulated data, for instance, created using InSilicoTEM\n"
" (note: simulated data accepted in special circumstances such\n"
" as test/training sets for validation challenges: you need to\n"
" ask for and be granted permission PRIOR to deposition otherwise\n"
" the dataset will be rejected by EMPIAR)\n"
"3 - raw image data relating to structures deposited to the Electron Microscopy Data Bank\n"
"4 - image data collected using serial block-face scanning electron microscopy \n"
" (like the Gatan 3View system)\n"
"5 - image data collected using focused ion beam scanning electron microscopy\n"
"6 - integrative hybrid modelling data.")
form.addParam(
'releaseDate',
params.EnumParam,
label="Release date",
condition="not resume",
choices=self._releaseDateTypes,
default=2,
important=True,
help="EMPIAR release date:\n"
"Options for releasing entry to the public: \n"
"IM - directly after the submission has been processed\n"
"EP - after the related EMDB entry has been released\n"
"HP - after the related primary citation has been published\n"
"H1 - delay release of entry by one year from the date of deposition"
)
form.addSection(label='Image sets')
self.inputSetsParam = form.addParam(
'inputSets',
params.MultiPointerParam,
label="Input set",
important=True,
condition="not resume",
pointerClass='EMSet,Volume',
minNumObjects=1,
help='Select one set (of micrographs, particles,'
' volumes, etc.) to be deposited to EMPIAR.')
# form.addParam('micSet', params.PointerParam, pointerClass='SetOfMicrographs,SetOfMovies,SetOfParticles',
# label='Image set', important=False,
# help='Image set to be uploaded to EMPIAR\n')
form.addSection(label="Principal investigator")
form.addParam(
'piFirstName',
params.StringParam,
label='First name',
condition="not resume",
help=
"PI first name e.g. Juan- this should not be empty if not using a custom template."
)
form.addParam(
'piLastName',
params.StringParam,
label='Last name',
condition="not resume",
help=
'PI Last name e.g. Perez - this should not be empty if not using a custom template.'
)
form.addParam(
'piOrg',
params.StringParam,
label='organization',
condition="not resume",
help=
"The name of the organization e.g. Biocomputing Unit, CNB-CSIC \n"
"This should not be empty if not using a custom template.")
form.addParam(
'piEmail',
params.StringParam,
label="Email",
condition="not resume",
help='PI Email address e.g. [email protected] - '
'this should not be empty if not using a custom template.')
form.addParam(
'piCountry',
params.StringParam,
label="Country",
condition="not resume",
help=
"Two letter country code eg. ES. This should not be empty if not using a custom template."
"\nValid country codes are %s" % " ".join(self._countryCodes))
form.addSection(label="Corresponding Author")
form.addParam(
'caFirstName',
params.StringParam,
label='First name',
condition="not resume",
help="Corresponding author's first name e.g. Juan. "
"This should not be empty if not using a custom template. ")
form.addParam(
'caLastName',
params.StringParam,
label='Last name',
condition="not resume",
help="Corresponding author's Last name e.g. Perez. "
"This should not be empty if not using a custom template.")
form.addParam(
'caOrg',
params.StringParam,
label='organization',
condition="not resume",
help="The name of the organization e.g. Biocomputing Unit, CNB-CSIC."
"This should not be empty if not using a custom template.")
form.addParam(
'caEmail',
params.StringParam,
label="Email",
condition="not resume",
help="Corresponding author's Email address e.g. [email protected]. "
"This should not be empty if not using a custom template.")
form.addParam(
'caCountry',
params.StringParam,
label="Country",
condition="not resume",
help=
"Two letter country code e.g. ES. This should not be empty if not using a custom template."
"\nValid country codes are %s" % " ".join(self._countryCodes))
# --------------- INSERT steps functions ----------------
def _insertAllSteps(self):
self._insertFunctionStep('createDepositionStep')
if self.submit:
self._insertFunctionStep('submitDepositionStep')
# --------------- STEPS functions -----------------------
def createDepositionStep(self):
# make folder in extra
if not self.resume:
pwutils.makePath(self._getExtraPath(self.entryTopLevel.get()))
# export workflow json
self.exportWorkflow()
# create deposition json
jsonTemplatePath = self.jsonTemplate.get(
'').strip() or self.getJsonTemplatePath()
entryAuthorStr = self.entryAuthor.get().split(',')
self.entryAuthorStr = "'%s', '%s'" % (entryAuthorStr[0].strip(),
entryAuthorStr[1].strip())
self.releaseDate = self._releaseDateTypes[self.releaseDate.get()]
self.experimentType = self.experimentType.get() + 1
jsonStr = open(jsonTemplatePath, 'rb').read().decode('utf-8')
jsonStr = jsonStr % self.__dict__
depoDict = json.loads(jsonStr)
imageSets = self.processImageSets()
depoDict[self.IMGSET_KEY] = imageSets
depoJson = self.getTopLevelPath(self.OUTPUT_DEPO_JSON)
with open(depoJson, 'w') as f:
# f.write(jsonStr.encode('utf-8'))
json.dump(depoDict, f, indent=4)
# self.depositionJsonPath = depoJson
self.depositionJsonPath.set(depoJson)
else:
self.depositionJsonPath.set(self.depositionJson.get())
with open(self.depositionJson.get()) as f:
depoDict = json.load(f)
self._store()
self.validateDepoJson(depoDict)
def submitDepositionStep(self):
depositorCall = '%(resume)s %(token)s %(depoJson)s %(ascp)s %(dataDir)s'
args = {
'resume':
'-r %s %s' % (self.entryID, self.uniqueDir) if self.resume else "",
'token':
os.environ[EMPIAR_TOKEN],
'depoJson':
os.path.abspath(self.depositionJsonPath.get()),
'ascp':
os.environ[ASCP_PATH],
'dataDir':
os.path.abspath(self.getTopLevelPath())
}
depositorCall = depositorCall % args
print("Empiar depositor call: %s" % depositorCall)
empiar_depositor.main(depositorCall.split())
# --------------- INFO functions -------------------------
def _validate(self):
errors = []
if self.submit:
if EMPIAR_TOKEN not in os.environ:
errors.append(
"Environment variable %s not set. Please set your %s in ~/.config/scipion/scipion.conf "
"or in your environment." % (EMPIAR_TOKEN, EMPIAR_TOKEN))
if ASPERA_PASS not in os.environ:
errors.append(
"Environment variable %s not set. Please set your %s in ~/.config/scipion/scipion.conf "
"or in your environment." % (ASPERA_PASS, ASPERA_PASS))
return errors
def _citations(self):
return ['Iudin2016']
def _summary(self):
summary = []
if self.depositionJsonPath.get():
summary.append('Generated deposition files:')
summary.append('- [[%s][Scipion workflow]]' % self.workflowPath)
summary.append('- [[%s][Deposition json]]' %
self.depositionJsonPath)
else:
summary.append('No deposition files generated yet')
return summary
def _methods(self):
return []
# -------------------- UTILS functions -------------------------
def getTopLevelPath(self, *paths):
return os.path.join(self._getExtraPath(self.entryTopLevel.get()),
*paths)
def getJsonTemplatePath(self):
return os.path.join(os.path.dirname(os.path.abspath(__file__)),
self.DEPOSITION_TEMPLATE)
def exportWorkflow(self):
project = self.getProject()
workflowProts = [p for p in project.getRuns()
] # workflow prots are all prots if no json provided
workflowJsonPath = os.path.join(
project.path, self.getTopLevelPath(self.OUTPUT_WORKFLOW))
protDicts = project.getProtocolDicts(workflowProts)
for inputSetPointer in self.inputSets:
inputSet = inputSetPointer.get()
setName = inputSet.getObjName()
setParentId = inputSet.getObjParentId()
setParentObj = project.getObject(setParentId)
protDicts[setParentId]['filesPath'] = os.path.join('.', setName)
pwutils.createLink(setParentObj._getExtraPath(),
self.getTopLevelPath(setName))
with open(workflowJsonPath, 'w') as f:
f.write(
json.dumps(list(protDicts.values()),
indent=4,
separators=(',', ': ')))
self.workflowPath.set(workflowJsonPath)
return workflowJsonPath
def validateDepoJson(self, depoDict):
with open(
os.path.join(os.path.dirname(__file__),
self.DEPOSITION_SCHEMA)) as f:
schema = json.load(f)
valid = jsonschema.validate(depoDict,
schema) # raises exception if not valid
return True
# --------------- imageSet utils -------------------------
def getEmpiarCategory(self, imageSet):
"""TODO"""
className = imageSet.getClassName()
category = self._imageSetCategories.get(className, 'OT')
if category == 'OT':
return 'OT', 'TODO'
else:
return category, ''
def getEmpiarFormat(self, imagePath):
ext = pwutils.getExt(imagePath).lower().strip('.')
imgFormat = self._imageSetFormats.get(ext, 'OT')
if imgFormat == 'OT':
return 'OT', ext
else:
return imgFormat, ''
def getVoxelType(self, imageObj):
"""TODO"""
dataType = self._ih.getDataType(imageObj)
empiarType = self._voxelTypes.get(dataType, 'OT')
if empiarType == 'OT':
return 'OT', 'TODO'
else:
return empiarType, ''
def getImageSetDict(self, imageSet):
firstImg = imageSet.getFirstItem()
firstFileName = firstImg.getFileName()
dims = imageSet.getDimensions()
micSetDict = copy.deepcopy(self._imageSetTemplate)
micSetDict[self.IMGSET_NAME] = imageSet.getObjName()
micSetDict[self.IMGSET_DIR] = "/data/%s" % imageSet.getObjName()
micSetDict[self.IMGSET_CAT] = "('%s', '%s')" % self.getEmpiarCategory(
imageSet)
micSetDict[
self.IMGSET_HEADER_FORMAT] = "('%s', '%s')" % self.getEmpiarFormat(
firstFileName)
micSetDict[
self.IMGSET_DATA_FORMAT] = "('%s', '%s')" % self.getEmpiarFormat(
firstFileName)
micSetDict[self.IMGSET_SIZE] = len(imageSet)
micSetDict[self.IMGSET_FRAMES] = dims[2]
micSetDict[
self.
IMGSET_VOXEL_TYPE] = "('%s', '%s')" % self.getVoxelType(firstImg)
micSetDict[self.IMGSET_PIXEL_WIDTH] = imageSet.getSamplingRate()
micSetDict[self.IMGSET_PIXEL_HEIGHT] = imageSet.getSamplingRate()
micSetDict[self.IMGSET_DETAILS] = "/data/%s" % os.path.basename(
self.workflowPath.get())
micSetDict[self.IMGSET_WIDTH] = dims[0]
micSetDict[self.IMGSET_HEIGHT] = dims[1]
return micSetDict
def processImageSets(self):
inputSets = [s.get() for s in self.inputSets]
imgSetDicts = []
for imgSet in inputSets:
imgSetDict = self.getImageSetDict(imgSet)
imgSetDicts.append(imgSetDict)
return imgSetDicts
class PKDepositionParameters(EMObject):
def __init__(self, **args):
EMObject.__init__(self, **args)
self.fnSubstance = String()
self.fnLung = String()
self.fnDeposition = String()
self.doseMultiplier = 1
def setFiles(self, fnSubstance, fnLung, fnDeposition):
self.fnSubstance.set(fnSubstance)
self.fnLung.set(fnLung)
self.fnDeposition.set(fnDeposition)
def readDepositionFile(self, alvlim):
fh=open(self.fnDeposition.get())
state = 0
for line in fh.readlines():
if line.strip()=="":
continue
if state == 0: # Header
tokens = line.split(':')
if tokens[0]=='Total dose [ug]':
self.dose = float(tokens[1].strip())*self.doseMultiplier
elif tokens[0]=='Diameter [um]':
self.diameterMode = tokens[1].strip()
elif 'FractionDeposited' in tokens[0]:
diameters = []
oldGeneration = 0
fractionMatrix = []
fractionRow = []
state = 1
elif state == 1: # table of numbers
tokens = [float(x.strip()) for x in line.split()]
diam = tokens[0]
generation = tokens[1]
fractionDeposited = tokens[2]
if not diam in diameters:
diameters.append(diam)
if oldGeneration!=generation:
if oldGeneration!=0:
fractionMatrix.append(fractionRow)
fractionRow=[]
oldGeneration=generation
fractionRow.append(fractionDeposited)
fractionMatrix.append(fractionRow)
fh.close()
diameters = np.asarray(diameters) # [um] for D
if self.diameterMode=="aerodynamic":
# Hartung2020_MATLAB/functions/aero2geom.m
lambdaVar = 1; # spherical shape
rho_water = 1; # [g / mL]
rho_subst = self.substance.rho * self.substance.MW * 1e-9; # [nmol / mL] *[g / mol] ->[g / mL]
diameters = diameters * np.sqrt( lambdaVar * rho_water / rho_subst);
self.dose_nmol = self.dose * 1e3 / self.substance.MW # % [ug] *1e3 / ([g/mol]) = [nmol]
self.bronchiDose_nmol = np.asarray(fractionMatrix[0:(alvlim-1)])*self.dose_nmol
# This is a matrix with as many rows as generations and as many columns as diameters
# The content is what is the dose deposited at that generation in nmol
alveolarMatrix = np.reshape(fractionMatrix[alvlim-1:],(len(fractionMatrix)-alvlim+1,diameters.size))
self.alveolarDose_nmol = np.sum(alveolarMatrix*self.dose_nmol, axis=0)
# This is a matrix with as many rows as generations in the alveoli and as many columns as diameters
# The content is what is the dose deposited at that generation in nmol
self.throatDose = self.dose_nmol - np.sum(self.bronchiDose_nmol) - np.sum(self.alveolarDose_nmol)
self.particleSize = diam2vol(diameters) # cm3
def read(self):
self.substance = PKSubstanceLungParameters()
self.substance.read(self.fnSubstance.get())
self.lung = PKPhysiologyLungParameters()
self.lung.read(self.fnLung.get())
alveolarGeneration = len(self.lung.getBronchial()['type'])+1
self.readDepositionFile(alveolarGeneration)
def getData(self):
data = {}
data['bronchial'] = self.bronchiDose_nmol
data['alveolar'] = self.alveolarDose_nmol
data['throat'] = self.throatDose
data['size'] = self.particleSize
data['dose_nmol'] = self.dose_nmol
return data
class PKPhysiologyLungParameters(EMObject):
def __init__(self, **args):
EMObject.__init__(self, **args)
self.fnPhys = String()
self.multiplier = [1] * 9
def write(self, fnOut):
fh=open(fnOut,"w")
fh.write("%f # cardiac output [mL/min]\n"%self.Qco)
fh.write("%f # lung tissue weight, without blood [g]\n"%self.OWlun)
fh.write("%f # alveolar fraction of cardiac output\n"%self.falvCO)
fh.write("%f # alveolar ELF volume [mL]\n"%self.Velf_alv)
# ELF = Epithelial Lining Fluid
fh.write("%f # alveolar surface area [cm2]\n"%self.Surf_alv)
fh.write("%f # bronchial fraction of cardiac output\n"%self.fbrCO)
fh.write("%f # bronchial fraction of lung tissue volume\n"%self.fbrVlun)
fh.write("%f # bronchial ELF heights for interpolation, trachea [cm]\n" % self.helf_trach)
fh.write("%f # bronchial ELF heights for interpolation, terminal bronchioles [cm]\n" % self.helf_termbr)
fh.write("%s # trachea length [cm]\n" % self.tracheaLength)
fh.write("%s # trachea diameter [cm]\n" % self.tracheaDiameter)
fh.write("%s # main bronchi length [cm]\n" % self.bronchi1Length)
fh.write("%s # main bronchi diameter [cm]\n" % self.bronchi1Diameter)
fh.write("%s # bronchi length [cm]\n" % self.bronchi2Length)
fh.write("%s # bronchi diameter [cm]\n" % self.bronchi2Diameter)
fh.write("%s # bronchiole length [cm]\n" % self.bronchi3Length)
fh.write("%s # bronchiole diameter [cm]\n" % self.bronchi3Diameter)
fh.write("%s # terminal bronchiole length [cm]\n" % self.bronchi4Length)
fh.write("%s # terminal bronchiole diameter [cm]\n" % self.bronchi4Diameter)
fh.close()
self.fnPhys.set(fnOut)
def read(self, fnIn):
fh=open(fnIn)
self.Qco=float(fh.readline().split()[0])
self.OWlun=float(fh.readline().split()[0])
self.falvCO=float(fh.readline().split()[0])
self.Velf_alv=float(fh.readline().split()[0])
self.Surf_alv=float(fh.readline().split()[0])
self.fbrCO=float(fh.readline().split()[0])
self.fbrVlun=float(fh.readline().split()[0])
self.helf_trach=float(fh.readline().split()[0])
self.helf_termbr=float(fh.readline().split()[0])
self.tracheaLength=fh.readline().split()[0]
self.tracheaDiameter=fh.readline().split()[0]
self.bronchi1Length=fh.readline().split()[0]
self.bronchi1Diameter=fh.readline().split()[0]
self.bronchi2Length=fh.readline().split()[0]
self.bronchi2Diameter=fh.readline().split()[0]
self.bronchi3Length=fh.readline().split()[0]
self.bronchi3Diameter=fh.readline().split()[0]
self.bronchi4Length=fh.readline().split()[0]
self.bronchi4Diameter=fh.readline().split()[0]
fh.close()
self.fnPhys.set(fnIn)
def getSystemic(self):
# Hartung2020_MATLAB/physiol_subst/physiology_systemic.m
data = {}
data['Qco']=self.Qco
data['OWlung']=self.OWlun
return data
def getBronchial(self):
# Hartung2020_MATLAB/physiol_subst/physiology_bronchial.m
def listSplit(valuesStr):
return [float(x) for x in valuesStr.split(',')]
tracheaLength = float(self.tracheaLength)
tracheaDiameter = float(self.tracheaDiameter)
bronchi1Length = float(self.bronchi1Length)
bronchi1Diameter = float(self.bronchi1Diameter)
bronchi2Length = listSplit(self.bronchi2Length)
bronchi2Diameter = listSplit(self.bronchi2Diameter)
bronchi3Length = listSplit(self.bronchi3Length)
bronchi3Diameter = listSplit(self.bronchi3Diameter)
bronchi4Length = float(self.bronchi4Length)
bronchi4Diameter = float(self.bronchi4Diameter)
segmentLengths = [tracheaLength, bronchi1Length] +bronchi2Length +bronchi3Length +[bronchi4Length]
segmentDiameters = [tracheaDiameter, bronchi1Diameter]+bronchi2Diameter+bronchi3Diameter+[bronchi4Diameter]
segmentType = ['T','B1'] + ['B2']*len(bronchi2Length) + ['B3']*len(bronchi3Length) + ['B4']
length_cm = np.asarray(segmentLengths)
diam_cm = np.asarray(segmentDiameters)
generation = np.arange(0,len(segmentLengths))+1
number = np.power(2.0,generation-1)
xArea_cm2 = math.pi*np.multiply(np.power(diam_cm/2,2.0),number) # cm2
vol_cm3 = np.multiply(xArea_cm2,length_cm) # cm3
start_cm = np.insert(np.cumsum(segmentLengths[:-1]),0,0,axis=0)
end_cm = np.cumsum(segmentLengths)
pos = 0.5*(start_cm+end_cm)
surf_cm2 = np.multiply(np.multiply(length_cm,(math.pi * diam_cm)),number)
h_elf_cm = np.interp(pos, [0, np.sum(length_cm)], [self.helf_trach, self.helf_termbr])
elf_cm3 = math.pi/4 * (np.power(diam_cm,2)-np.power((diam_cm-2*h_elf_cm),2))
elf_cm3 = np.multiply(np.multiply(elf_cm3,length_cm),number)
Vtis_ctr = self.OWlun*self.fbrVlun; # central lung tissue volume
voltis_cm3 = elf_cm3 * Vtis_ctr / sum(elf_cm3);
data = {}
data['generation']=generation
data['type']=segmentType
data['number']=number
data['length_cm']=length_cm
data['diam_cm']=diam_cm
data['xArea_cm2']=xArea_cm2
data['vol_cm3']=vol_cm3
data['start_cm']=start_cm
data['end_cm']=end_cm
data['pos']=pos
data['surf_cm2']=surf_cm2 * self.multiplier[5]
data['h_elf_trach'] = self.helf_trach
data['h_elf_termbr'] = self.helf_termbr
data['h_elf_cm'] = h_elf_cm
data['elf_cm3'] = elf_cm3 * self.multiplier[1]
data['voltis_cm3'] = voltis_cm3 * self.multiplier[3]
data['fVol'] = self.fbrVlun
data['fQco'] = self.fbrCO * self.multiplier[7]
return data
def getAlveolar(self):
# Hartung2020_MATLAB/physiol_subst/physiology_alveolar.m
data = {}
data['fQco']=self.falvCO * self.multiplier[8]
data['fVol']=1-self.fbrVlun
data['ELF_cm3'] = self.Velf_alv * self.multiplier[2]
data['Vol_cm3'] = self.OWlun * data['fVol'] * self.multiplier[4] # density = 1 [g/cm^3]
data['Surf_cm2'] = self.Surf_alv * self.multiplier[6]
return data
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
INPUT_HELP = """ Input volume(s) for ResMap.
Required volume properties:
1. The particle must be centered in the volume.
2. The background must not been masked out.
Desired volume properties:
1. The volume has not been filtered in any way (low-pass filtering, etc.)
2. The volume has a realistic noise spectrum.
This is sometimes obtained by so-called amplitude correction.
While a similar effect is often obtained by B-factor sharpening,
please make sure that the spectrum does not blow up near Nyquist.
"""
@classmethod
def validateInstallation(cls):
""" This function will be used to check if package is properly installed."""
missingPaths = ["%s: %s" % (var, os.environ[var])
for var in [RESMAP_HOME]
if not os.path.exists(os.environ[var])]
if missingPaths:
return ["Missing variables:"] + missingPaths
else:
return [] # No errors
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
self.plotData = String() # store some values for later plot
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('useSplitVolume', params.BooleanParam, default=False,
label="Use half volumes?",
help='Use split volumes for gold-standard FSC.')
form.addParam('inputVolume', params.PointerParam,
pointerClass='Volume', condition="not useSplitVolume",
label="Input volume", important=True,
help=self.INPUT_HELP)
form.addParam('volumeHalf1', params.PointerParam,
label="Volume half 1", important=True,
pointerClass='Volume', condition="useSplitVolume",
help=self.INPUT_HELP)
form.addParam('volumeHalf2', params.PointerParam,
pointerClass='Volume', condition="useSplitVolume",
label="Volume half 2", important=True,
help=self.INPUT_HELP)
form.addParam('applyMask', params.BooleanParam, default=False,
label="Mask input volume?",
help="It is not necessary to provide ResMap with a mask "
"volume. The algorithm will attempt to estimate a "
"mask volume by low-pass filtering the input volume "
"and thresholding it using a heuristic procedure.\n"
"If the automated procedure does not work well for "
"your particle, you may provide a mask volume that "
"matches the input volume in size and format. "
"The mask volume should be a binary volume with zero "
"(0) denoting the background/solvent and some positive"
"value (0+) enveloping the particle.")
form.addParam('maskVolume', params.PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
form.addParam('whiteningLabel', params.LabelParam, important=True,
label="It is strongly recommended to use the "
"pre-whitening wizard.")
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', params.FloatParam, default=10,
label="Angstroms")
line.addParam('prewhitenRamp', params.FloatParam, default=1,
label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', params.FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help="Default (0): algorithm will start a just above\n"
" 2*voxelSize until 4*voxelSize. \n"
"These fields are provided to accelerate computation "
"if you are only interested in analyzing a specific "
"resolution range. It is usually a good idea to provide "
"a maximum resolution value to save time. Another way to "
"save computation is to provide a larger step size.")
line.addParam('minRes', params.FloatParam, default=0, label='Min')
line.addParam('maxRes', params.FloatParam, default=0, label='Max')
group.addParam('pVal', params.FloatParam, default=0.05,
label='Confidence level:',
help="P-value, usually between [0.01, 0.05].\n\n"
"This is the p-value of the statistical hypothesis test "
"on which ResMap is based on. It is customarily set to "
"0.05 although you are welcome to reduce it (e.g. 0.01) "
"if you would like to obtain a more conservative result. "
"Empirically, ResMap results are not much affected by the p-value.")
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
if self.useSplitVolume:
inputs = [self.volumeHalf1, self.volumeHalf2]
self.inputVolume.set(None)
else:
inputs = [self.inputVolume]
self.volumeHalf1.set(None)
self.volumeHalf2.set(None)
locations = [i.get().getLocation() for i in inputs]
self._insertFunctionStep('convertInputStep', *locations)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropriate parameters. """
results = self.runResmap(self._getPath())
self.histogramData.set(dumps(results['resHisto']))
plotDict = {'minRes': results['minRes'],
'maxRes': results['maxRes'],
'orig_n': results['orig_n'],
'n': results['n'],
'currentRes': results['currentRes']
}
self.plotData.set(dumps(plotDict))
self._store(self.histogramData, self.plotData)
self.savePlots(results)
def savePlots(self, results=None):
""" Store png images of the plots to be used as images, """
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
self._plotVolumeSlices().savefig(self._getExtraPath('volume1.map.png'))
plot = self._plotResMapSlices(results['resTOTALma'])
plot.savefig(self._getExtraPath('volume1_resmap.map.png'))
self._plotHistogram().savefig(self._getExtraPath('histogram.png'))
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
if exists(self._getExtraPath('histogram.png')):
results = self._parseOutput()
summary.append('Mean resolution: %0.2f A' % results[0])
summary.append('Median resolution: %0.2f A' % results[1])
else:
summary.append("Output is not ready yet.")
return summary
def _validate(self):
errors = []
if self.useSplitVolume:
half1 = self.volumeHalf1.get()
half2 = self.volumeHalf2.get()
if half1.getSamplingRate() != half2.getSamplingRate():
errors.append('The selected half volumes have not the same pixel size.')
if half1.getXDim() != half2.getXDim():
errors.append('The selected half volumes have not the same dimensions.')
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ[RESMAP_HOME])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'vxSize': self.volumeHalf1.get().getSamplingRate(),
'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'vxSize': self.inputVolume.get().getSamplingRate(),
'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self, **kwargs):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'), **kwargs)
return Plotter(figure=fig)
def _plotResMapSlices(self, data=None, **kwargs):
from ResMap_visualization import plotResMapVolume
plotDict = loads(self.plotData.get())
if data is None:
data = self._getVolumeMatrix('volume1_resmap.map')
data = np.ma.masked_where(data > plotDict['currentRes'], data, copy=True)
kwargs.update(plotDict)
fig = plotResMapVolume(data, **kwargs)
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
def _parseOutput(self):
meanRes, medianRes = 0, 0
f = open(self.getLogPaths()[0], 'r')
for line in f.readlines():
if 'MEAN RESOLUTION in MASK' in line:
meanRes = line.strip().split('=')[1]
elif 'MEDIAN RESOLUTION in MASK' in line:
medianRes = line.strip().split('=')[1]
f.close()
return tuple(map(float, (meanRes, medianRes)))
class XmippProtDimredNMA(ProtAnalysis3D):
""" This protocol will take the images with NMA deformations
as points in a N-dimensional space (where N is the number
of computed normal modes) and will project them in a reduced
spaced (usually with less dimensions).
"""
_label = 'nma dimred'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.mappingFile = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputNMA',
PointerParam,
pointerClass='XmippProtAlignmentNMA',
label="Conformational distribution",
help='Select a previous run of the NMA alignment.')
form.addParam('dimredMethod',
EnumParam,
default=DIMRED_PCA,
choices=[
'Principal Component Analysis (PCA)',
'Local Tangent Space Alignment', 'Diffusion map',
'Linear Local Tangent Space Alignment',
'Linearity Preserving Projection', 'Kernel PCA',
'Probabilistic PCA', 'Laplacian Eigenmap',
'Hessian Locally Linear Embedding',
'Stochastic Proximity Embedding',
'Neighborhood Preserving Embedding'
],
label='Dim-Red method',
help=""" Dimensionality Reduction method.
PCA
Principal Component Analysis
LTSA <k=12>
Local Tangent Space Alignment, k=number of nearest neighbours
DM <s=1> <t=1>
Diffusion map, t=Markov random walk, s=kernel sigma
LLTSA <k=12>
Linear Local Tangent Space Alignment, k=number of nearest neighbours
LPP <k=12> <s=1>
Linearity Preserving Projection, k=number of nearest neighbours, s=kernel sigma
kPCA <s=1>
Kernel PCA, s=kernel sigma
pPCA <n=200>
Probabilistic PCA, n=number of iterations
LE <k=7> <s=1>
Laplacian Eigenmap, k=number of nearest neighbours, s=kernel sigma
HLLE <k=12>
Hessian Locally Linear Embedding, k=number of nearest neighbours
SPE <k=12> <global=1>
Stochastic Proximity Embedding, k=number of nearest neighbours, global embedding or not
NPE <k=12>
Neighborhood Preserving Embedding, k=number of nearest neighbours
""")
form.addParam('extraParams',
StringParam,
level=LEVEL_ADVANCED,
label="Extra params",
help='This parameters will be passed to the program.')
form.addParam('reducedDim',
IntParam,
default=2,
label='Reduced dimension')
form.addParallelSection(threads=0, mpi=0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Take deforamtions text file and the number of images and modes
inputSet = self.getInputParticles()
rows = inputSet.getSize()
reducedDim = self.reducedDim.get()
method = self.dimredMethod.get()
extraParams = self.extraParams.get('')
deformationsFile = self.getDeformationFile()
self._insertFunctionStep('convertInputStep', deformationsFile,
inputSet.getObjId())
self._insertFunctionStep('performDimredStep', deformationsFile, method,
extraParams, rows, reducedDim)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, deformationFile, inputId):
""" Iterate through the images and write the
plain deformation.txt file that will serve as
input for dimensionality reduction.
"""
inputSet = self.getInputParticles()
f = open(deformationFile, 'w')
for particle in inputSet:
f.write(' '.join(particle._xmipp_nmaDisplacements))
f.write('\n')
f.close()
def performDimredStep(self, deformationsFile, method, extraParams, rows,
reducedDim):
outputMatrix = self.getOutputMatrixFile()
methodName = DIMRED_VALUES[method]
# Get number of columes in deformation files
# it can be a subset of inputModes
f = open(deformationsFile)
columns = len(
f.readline().split()) # count number of values in first line
f.close()
args = "-i %(deformationsFile)s -o %(outputMatrix)s -m %(methodName)s %(extraParams)s"
args += "--din %(columns)d --samples %(rows)d --dout %(reducedDim)d"
if method in DIMRED_MAPPINGS:
mappingFile = self._getExtraPath('projector.txt')
args += " --saveMapping %(mappingFile)s"
self.mappingFile.set(mappingFile)
self.runJob("xmipp_matrix_dimred", args % locals())
def createOutputStep(self):
pass
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
def _citations(self):
return []
def _methods(self):
return []
#--------------------------- UTILS functions --------------------------------------------
def getInputParticles(self):
""" Get the output particles of the input NMA protocol. """
return self.inputNMA.get().outputParticles
def getInputPdb(self):
return self.inputNMA.get().getInputPdb()
def getOutputMatrixFile(self):
return self._getExtraPath('output_matrix.txt')
def getDeformationFile(self):
return self._getExtraPath('deformations.txt')
def getProjectorFile(self):
return self.mappingFile.get()
def getMethodName(self):
return DIMRED_VALUES[self.dimredMethod.get()]
class HostConfig(OrderedObject):
""" Main store the configuration for execution hosts. """
def __init__(self, **kwargs):
OrderedObject.__init__(self, **kwargs)
self.label = String(kwargs.get('label', None))
self.hostName = String(kwargs.get('hostName', None))
self.userName = String()
self.password = String()
self.hostPath = String()
self.mpiCommand = String()
self.scipionHome = String()
self.scipionConfig = String()
self.address = String()
self.queueSystem = QueueSystemConfig()
def getLabel(self):
return self.label.get()
def getHostName(self):
return self.hostName.get()
def getUserName(self):
return self.userName.get()
def getPassword(self):
return self.password.get()
def getHostPath(self):
return self.hostPath.get()
def getSubmitCommand(self):
return self.queueSystem.submitCommand.get()
def getSubmitPrefix(self):
return self.queueSystem.submitPrefix.get()
def getCheckCommand(self):
return self.queueSystem.checkCommand.get()
def getCancelCommand(self):
return self.queueSystem.cancelCommand.get()
def isQueueMandatory(self):
return self.queueSystem.mandatory.get()
def getSubmitTemplate(self):
return self.queueSystem.getSubmitTemplate()
def getQueuesDefault(self):
return self.queueSystem.queuesDefault
def getMpiCommand(self):
return self.mpiCommand.get()
def getQueueSystem(self):
return self.queueSystem
def getJobDoneRegex(self):
return self.queueSystem.jobDoneRegex.get()
def setLabel(self, label):
self.label.set(label)
def setHostName(self, hostName):
self.hostName.set(hostName)
def setUserName(self, userName):
self.userName.set(userName)
def setPassword(self, password):
self.password.set(password)
def setHostPath(self, hostPath):
self.hostPath.set(hostPath)
def setMpiCommand(self, mpiCommand):
self.mpiCommand.set(mpiCommand)
def setQueueSystem(self, queueSystem):
self.queueSystem = queueSystem
def getScipionHome(self):
""" Return the path where Scipion is installed in
the host. This is useful when launching remote jobs.
"""
return self.scipionHome.get()
def setScipionHome(self, newScipionHome):
self.scipionHome.set(newScipionHome)
def getScipionConfig(self):
""" From which file to read the configuration file in
this hosts. Useful for remote jobs.
"""
return self.scipionConfig.get()
def setScipionConfig(self, newConfig):
self.scipionConfig.set(newConfig)
def getAddress(self):
return self.address.get()
def setAddress(self, newAddress):
return self.address.set(newAddress)
@classmethod
def writeBasic(cls, configFn):
""" Write a very basic Host configuration for testing purposes. """
with open(configFn, 'w') as f:
f.write('[localhost]\nPARALLEL_COMMAND = '
'mpirun -np %%(JOB_NODES)d --map-by node %%(COMMAND)s\n')
@classmethod
def load(cls, hostsConf):
""" Load several hosts from a configuration file.
Return an dictionary with hostName -> hostConfig pairs.
"""
# Read from users' config file. Raw to avoid interpolation of %: we expect %_
cp = RawConfigParser(comment_prefixes=";")
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
hosts = OrderedDict()
try:
assert cp.read(hostsConf) != [], 'Missing file %s' % hostsConf
for hostName in cp.sections():
host = HostConfig(label=hostName, hostName=hostName)
host.setHostPath(pw.Config.SCIPION_USER_DATA)
# Helper functions (to write less)
def get(var, default=None):
if cp.has_option(hostName, var):
value = cp.get(hostName, var)
# Rescue python2.7 behaviour: ## at the beginning of a line, means a single #.
# https://github.com/scipion-em/scipion-pyworkflow/issues/70
value = value.replace("\n##", "\n#")
# Keep compatibility: %_ --> %%
value = value.replace('%_(', '%(')
return value
else:
return default
def getDict(var):
od = OrderedDict()
if cp.has_option(hostName, var):
for key, value in json.loads(get(var)).items():
od[key] = value
return od
host.setScipionHome(
get(pw.SCIPION_HOME_VAR, pw.Config.SCIPION_HOME))
host.setScipionConfig(pw.Config.SCIPION_CONFIG)
# Read the address of the remote hosts,
# using 'localhost' as default for backward compatibility
host.setAddress(get('ADDRESS', 'localhost'))
host.mpiCommand.set(get('PARALLEL_COMMAND'))
host.queueSystem = QueueSystemConfig()
hostQueue = host.queueSystem # shortcut
hostQueue.name.set(get('NAME'))
# If the NAME is not provided or empty
# do no try to parse the rest of Queue parameters
if hostQueue.hasName():
hostQueue.setMandatory(get('MANDATORY', 0))
hostQueue.submitPrefix.set(get('SUBMIT_PREFIX', ''))
hostQueue.submitCommand.set(get('SUBMIT_COMMAND'))
hostQueue.submitTemplate.set(get('SUBMIT_TEMPLATE'))
hostQueue.cancelCommand.set(get('CANCEL_COMMAND'))
hostQueue.checkCommand.set(get('CHECK_COMMAND'))
hostQueue.jobDoneRegex.set(get('JOB_DONE_REGEX'))
hostQueue.queues = getDict('QUEUES')
hostQueue.queuesDefault = getDict('QUEUES_DEFAULT')
hosts[hostName] = host
return hosts
except Exception as e:
sys.exit('Failed to read settings. The reported error was:\n %s\n'
'To solve it, delete %s and run again.' %
(e, os.path.abspath(hostsConf)))
class QueueSystemConfig(OrderedObject):
def __init__(self, **kwargs):
OrderedObject.__init__(self, **kwargs)
self.name = String()
# Number of cores from which the queue is mandatory
# 0 means no mandatory at all
# 1 will force to launch all jobs through the queue
self.mandatory = Integer()
self.queues = None # List for queue configurations
self.submitCommand = String()
# Allow to change the prefix of submission scripts
# we used by default the ID.job, but in some clusters
# the job script should start by a letter
self.submitPrefix = String()
self.checkCommand = String()
self.cancelCommand = String()
self.submitTemplate = String()
self.jobDoneRegex = String()
def hasName(self):
return self.name.hasValue()
def hasValue(self):
return self.hasName() and len(self.queues)
def getName(self):
return self.name.get()
def getMandatory(self):
return self.mandatory.get()
def getSubmitTemplate(self):
return self.submitTemplate.get()
def getSubmitCommand(self):
return self.submitCommand.get()
def getCheckCommand(self):
return self.checkCommand.get()
def getCancelCommand(self):
return self.cancelCommand.get()
def getQueues(self):
return self.queues
def setName(self, name):
self.name.set(name)
def setMandatory(self, mandatory):
# This condition is to be backward compatible
# when mandatory was a boolean
# now it should use the number of CPU
# that should force to use the queue
if mandatory in ['False', 'false']:
mandatory = 0
elif mandatory in ['True', 'true']:
mandatory = 1
self.mandatory.set(mandatory)
def setSubmitTemplate(self, submitTemplate):
self.submitTemplate.set(submitTemplate)
def setSubmitCommand(self, submitCommand):
self.submitCommand.set(submitCommand)
def setCheckCommand(self, checkCommand):
self.checkCommand.set(checkCommand)
def setCancelCommand(self, cancelCommand):
self.cancelCommand.set(cancelCommand)
def setJobDoneRegex(self, jobDoneRegex):
self.jobDoneRegex.set(jobDoneRegex)
def setQueues(self, queues):
self.queues = queues
def getQueueConfig(self, objId):
if objId is not None and self.queues is not None:
for queueConfig in self.queues:
if objId == queueConfig.getObjId():
return queueConfig
return None
class XmippProtDimredNMA(ProtAnalysis3D):
""" This protocol will take the images with NMA deformations
as points in a N-dimensional space (where N is the number
of computed normal modes) and will project them in a reduced
spaced (usually with less dimensions).
"""
_label = 'nma dimred'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.mappingFile = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputNMA', PointerParam, pointerClass='XmippProtAlignmentNMA',
label="Conformational distribution",
help='Select a previous run of the NMA alignment.')
form.addParam('dimredMethod', EnumParam, default=DIMRED_PCA,
choices=['Principal Component Analysis (PCA)',
'Local Tangent Space Alignment',
'Diffusion map',
'Linear Local Tangent Space Alignment',
'Linearity Preserving Projection',
'Kernel PCA',
'Probabilistic PCA',
'Laplacian Eigenmap',
'Hessian Locally Linear Embedding',
'Stochastic Proximity Embedding',
'Neighborhood Preserving Embedding'],
label='Dim-Red method',
help=""" Dimensionality Reduction method.
PCA
Principal Component Analysis
LTSA <k=12>
Local Tangent Space Alignment, k=number of nearest neighbours
DM <s=1> <t=1>
Diffusion map, t=Markov random walk, s=kernel sigma
LLTSA <k=12>
Linear Local Tangent Space Alignment, k=number of nearest neighbours
LPP <k=12> <s=1>
Linearity Preserving Projection, k=number of nearest neighbours, s=kernel sigma
kPCA <s=1>
Kernel PCA, s=kernel sigma
pPCA <n=200>
Probabilistic PCA, n=number of iterations
LE <k=7> <s=1>
Laplacian Eigenmap, k=number of nearest neighbours, s=kernel sigma
HLLE <k=12>
Hessian Locally Linear Embedding, k=number of nearest neighbours
SPE <k=12> <global=1>
Stochastic Proximity Embedding, k=number of nearest neighbours, global embedding or not
NPE <k=12>
Neighborhood Preserving Embedding, k=number of nearest neighbours
""")
form.addParam('extraParams', StringParam, level=LEVEL_ADVANCED,
label="Extra params",
help='This parameters will be passed to the program.')
form.addParam('reducedDim', IntParam, default=2,
label='Reduced dimension')
form.addParallelSection(threads=0, mpi=0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Take deforamtions text file and the number of images and modes
inputSet = self.getInputParticles()
rows = inputSet.getSize()
reducedDim = self.reducedDim.get()
method = self.dimredMethod.get()
extraParams = self.extraParams.get('')
deformationsFile = self.getDeformationFile()
self._insertFunctionStep('convertInputStep',
deformationsFile, inputSet.getObjId())
self._insertFunctionStep('performDimredStep',
deformationsFile, method, extraParams,
rows, reducedDim)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, deformationFile, inputId):
""" Iterate throught the images and write the
plain deformation.txt file that will serve as
input for dimensionality reduction.
"""
inputSet = self.getInputParticles()
f = open(deformationFile, 'w')
for particle in inputSet:
f.write(' '.join(particle._xmipp_nmaDisplacements))
f.write('\n')
f.close()
def performDimredStep(self, deformationsFile, method, extraParams,
rows, reducedDim):
outputMatrix = self.getOutputMatrixFile()
methodName = DIMRED_VALUES[method]
# Get number of columes in deformation files
# it can be a subset of inputModes
f = open(deformationsFile)
columns = len(f.readline().split()) # count number of values in first line
f.close()
args = "-i %(deformationsFile)s -o %(outputMatrix)s -m %(methodName)s %(extraParams)s"
args += "--din %(columns)d --samples %(rows)d --dout %(reducedDim)d"
if method in DIMRED_MAPPINGS:
mappingFile = self._getExtraPath('projector.txt')
args += " --saveMapping %(mappingFile)s"
self.mappingFile.set(mappingFile)
self.runJob("xmipp_matrix_dimred", args % locals())
def createOutputStep(self):
pass
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
def _citations(self):
return []
def _methods(self):
return []
#--------------------------- UTILS functions --------------------------------------------
def getInputParticles(self):
""" Get the output particles of the input NMA protocol. """
return self.inputNMA.get().outputParticles
def getInputPdb(self):
return self.inputNMA.get().getInputPdb()
def getOutputMatrixFile(self):
return self._getExtraPath('output_matrix.txt')
def getDeformationFile(self):
return self._getExtraPath('deformations.txt')
def getProjectorFile(self):
return self.mappingFile.get()
def getMethodName(self):
return DIMRED_VALUES[self.dimredMethod.get()]
class HostConfig(OrderedObject):
""" Main store the configuration for execution hosts. """
def __init__(self, **kwargs):
OrderedObject.__init__(self, **kwargs)
self.label = String(kwargs.get('label', None))
self.hostName = String(kwargs.get('hostName', None))
self.userName = String()
self.password = String()
self.hostPath = String()
self.mpiCommand = String()
self.scipionHome = String()
self.scipionConfig = String()
self.address = String()
self.queueSystem = QueueSystemConfig()
def getLabel(self):
return self.label.get()
def getHostName(self):
return self.hostName.get()
def getUserName(self):
return self.userName.get()
def getPassword(self):
return self.password.get()
def getHostPath(self):
return self.hostPath.get()
def getSubmitCommand(self):
return self.queueSystem.submitCommand.get()
def getSubmitPrefix(self):
return self.queueSystem.submitPrefix.get()
def getCheckCommand(self):
return self.queueSystem.checkCommand.get()
def getCancelCommand(self):
return self.queueSystem.cancelCommand.get()
def isQueueMandatory(self):
return self.queueSystem.mandatory.get()
def getSubmitTemplate(self):
return self.queueSystem.getSubmitTemplate()
def getQueuesDefault(self):
return self.queueSystem.queuesDefault
def getMpiCommand(self):
return self.mpiCommand.get()
def getQueueSystem(self):
return self.queueSystem
def getJobDoneRegex(self):
return self.queueSystem.jobDoneRegex.get()
def setLabel(self, label):
self.label.set(label)
def setHostName(self, hostName):
self.hostName.set(hostName)
def setUserName(self, userName):
self.userName.set(userName)
def setPassword(self, password):
self.password.set(password)
def setHostPath(self, hostPath):
self.hostPath.set(hostPath)
def setMpiCommand(self, mpiCommand):
self.mpiCommand.set(mpiCommand)
def setQueueSystem(self, queueSystem):
self.queueSystem = queueSystem
def getScipionHome(self):
""" Return the path where Scipion is installed in
the host. This is useful when launching remote jobs.
"""
return self.scipionHome.get()
def setScipionHome(self, newScipionHome):
self.scipionHome.set(newScipionHome)
def getScipionConfig(self):
""" From which file to read the configuration file in
this hosts. Useful for remote jobs.
"""
return self.scipionConfig.get()
def setScipionConfig(self, newConfig):
self.scipionConfig.set(newConfig)
def getAddress(self):
return self.address.get()
def setAddress(self, newAddress):
return self.address.set(newAddress)
@classmethod
def load(cls, hostsConf):
""" Load several hosts from a configuration file.
Return an dictionary with hostName -> hostConfig pairs.
"""
# Read from users' config file.
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
hosts = OrderedDict()
try:
assert cp.read(hostsConf) != [], 'Missing file %s' % hostsConf
for hostName in cp.sections():
host = HostConfig(label=hostName, hostName=hostName)
host.setHostPath(pw.Config.SCIPION_USER_DATA)
# Helper functions (to write less)
def get(var, default=None):
if cp.has_option(hostName, var):
return cp.get(hostName, var).replace('%_(', '%(')
else:
return default
def getDict(var):
od = OrderedDict()
if cp.has_option(hostName, var):
for key, value in json.loads(get(var)).iteritems():
od[key] = value
return od
host.setScipionHome(get('SCIPION_HOME', pw.Config.SCIPION_HOME))
host.setScipionConfig(get('SCIPION_CONFIG'))
# Read the address of the remote hosts,
# using 'localhost' as default for backward compatibility
host.setAddress(get('ADDRESS', 'localhost'))
host.mpiCommand.set(get('PARALLEL_COMMAND'))
host.queueSystem = QueueSystemConfig()
hostQueue = host.queueSystem # shortcut
hostQueue.name.set(get('NAME'))
# If the NAME is not provided or empty
# do no try to parse the rest of Queue parameters
if hostQueue.hasName():
hostQueue.setMandatory(get('MANDATORY', 0))
hostQueue.submitPrefix.set(get('SUBMIT_PREFIX', ''))
hostQueue.submitCommand.set(get('SUBMIT_COMMAND'))
hostQueue.submitTemplate.set(get('SUBMIT_TEMPLATE'))
hostQueue.cancelCommand.set(get('CANCEL_COMMAND'))
hostQueue.checkCommand.set(get('CHECK_COMMAND'))
hostQueue.jobDoneRegex.set(get('JOB_DONE_REGEX'))
hostQueue.queues = getDict('QUEUES')
hostQueue.queuesDefault = getDict('QUEUES_DEFAULT')
hosts[hostName] = host
return hosts
except Exception as e:
sys.exit('Failed to read settings. The reported error was:\n %s\n'
'To solve it, delete %s and run again.' % (e, hostsConf))
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputVolume', PointerParam, pointerClass='Volume',
label="Input volume", important=True,
help='Select the input volume.')
form.addParam('useSplitVolume', BooleanParam, default=False,
label="Use split volume?",
help='Use to split volumes for gold-standard FSC.')
form.addParam('splitVolume', PointerParam, label="Split volume", important=True,
pointerClass='Volume', condition="useSplitVolume",
help='Select the second split volume.')
form.addParam('applyMask', BooleanParam, default=False,
label="Mask input volume?",
help='If set to <No> ResMap will automatically compute a mask.')
form.addParam('maskVolume', PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', FloatParam, default=0, label="Angstroms")
line.addParam('prewhitenRamp', FloatParam, default=0, label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help='Default: algorithm will start a just above 2*voxelSize')
line.addParam('minRes', FloatParam, default=0, label='Min')
line.addParam('maxRes', FloatParam, default=0, label='Max')
group.addParam('pVal', FloatParam, default=0.05,
label='Confidence level:',
help='usually between [0.01, 0.05]')
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
inputs = [self.inputVolume.get().getLocation()]
if self.useSplitVolume:
inputs.append(self.splitVolume.get().getLocation())
self._insertFunctionStep('convertInputStep', *inputs)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
self._insertFunctionStep('savePlotsStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropiate parameters. """
results = self.runResmap(self._getPath())
from cPickle import dumps
self.histogramData.set(dumps(results['resHisto']))
self._store(self.histogramData)
def savePlotsStep(self):
""" Store png images of the plots to be used
from web.
"""
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
self._plotVolumeSlices().savefig(self._getExtraPath('volume1.map.png'))
self._plotResMapSlices().savefig(self._getExtraPath('volume1_resmap.map.png'))
self._plotHistogram().savefig(self._getExtraPath('histogram.png'))
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'vxSize': self.inputVolume.get().getSamplingRate(),
'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'))
return Plotter(figure=fig)
def _plotResMapSlices(self):
from ResMap_visualization import plotResMapVolume
fig = plotResMapVolume(self._getVolumeMatrix('volume1_resmap.map'),
minRes=self.minRes.get(),
maxRes=self.maxRes.get())
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
from cPickle import loads
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
