def distribute_job(self, job, verbose=False): pl_filename = job["ParticleList"] from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(pl_filename) nn = len(pl) all_pairs = [] for i in range(nn): for j in range(i+1, nn): all_pairs.append((i, j)) n = len(all_pairs) particlesPerNode = int(n/self.num_workers) residual = n-particlesPerNode*self.num_workers start_idx = 0 for i in range(1, self.num_workers+1): if i < residual+1: # since i starts from 1 l = particlesPerNode+1 else: l = particlesPerNode sub_pairs = all_pairs[start_idx : start_idx+l] start_idx += l # construct the job sub_job = {} sub_job["Pairs"] = sub_pairs sub_job["ParticleList"] = pl_filename sub_job["Mask"] = job["Mask"] sub_job["Frequency"] = job["Frequency"] sub_job["Binning"] = job["Binning"] self.send_job(sub_job, i) if verbose: print(self.node_name + ': distributed %d particles to node %d' % (len(sub_pairs), i))
def get_phase_flip_pl(self): if self.phase_flip_pl_obj is None: from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(self.phase_flip_pl) self.phase_flip_pl_obj = pl return self.phase_flip_pl_obj
def start(self, job, outdir='./', verbose=False): outdir = job["outdir"] if self.mpi_id == 0: import numpy as np pl_filename = job["ParticleList"] from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(pl_filename) n = len(pl) correlation_matrix = np.ones((n, n)) # distribute the job self.distribute_job(job, verbose) # fill in the diagnal line for i in range(n): correlation_matrix[i][i] = 1 # gather the results for i in range(self.num_workers): result = self.get_result() pairs = list(result.keys()) for pair in pairs: correlation_matrix[pair[0]][pair[1]] = result[pair] correlation_matrix[pair[1]][pair[0]] = result[pair] # write the correlation matrix to the disk np.savetxt(os.path.join(outdir, 'correlation_matrix.csv'), correlation_matrix, delimiter=',') # send end signal to other nodes and terminate itself self.end(verbose) else: # other nodes self.run(verbose)
def writeCroppedParticles(particleListName, output, center, cubesize): """ @param particleListName: name of particle list @type particleListName: str @param output: Name of output particles @type output: str @param center: center of output particles in template orientation @type center: list @param cubesize: Size of output particles in pixel @type cubesize: int """ from pytom.basic.structures import ParticleList, Particle, Shift from pytom_volume import transformSpline as transform from pytom_volume import subvolume, vol pl = ParticleList() pl.fromXMLFile(filename=particleListName) #copy particle list for list of cropped particles pl_new = pl.copy() pvol = pl[0].getVolume() sizeX = pvol.sizeX() sizeY = pvol.sizeY() sizeZ = pvol.sizeZ() pvol_ali = vol(sizeX, sizeY, sizeZ) subV = vol(cubesize, cubesize, cubesize) sub_startX = center[0]-cubesize/2 sub_startY = center[1]-cubesize/2 sub_startZ = center[2]-cubesize/2 if (sub_startX < 0) or (sub_startY < 0) or (sub_startZ < 0): raise ValueError('cubesize too large :(') for (ipart, part) in enumerate(pl): pvol_ali.setAll(0) subV.setAll(0) pvol = part.getVolume() rot = part.getRotation() rotinvert = rot.invert() shiftV = part.getShift() transform(pvol, pvol_ali, rotinvert[0], rotinvert[1], rotinvert[2], sizeX/2, sizeY/2, sizeZ/2, -shiftV[0], -shiftV[1], -shiftV[2], 0, 0, 0) # box out subvolume subV = subvolume(pvol_ali, sub_startX, sub_startY, sub_startZ, cubesize, cubesize, cubesize) transform(subV, subV, rot[0], rot[1], rot[2], cubesize/2, cubesize/2, cubesize/2, 0, 0, 0, 0, 0, 0) fname = part.getFilename() idx = fname.split('_')[-1].split('.')[0] nfname = output+'_'+idx+'.em' print("write file " + nfname) subV.write(nfname) pl_new[ipart].setFilename(newFilename=nfname) pl_new[ipart].setShift(shift=Shift(0,0,0)) return pl_new
def cleanUp_RandomParticleList(pl_filename='pl.xml', pdir='./testparticles'): """ remove directories """ from os import remove, rmdir from pytom.basic.structures import ParticleList pl = ParticleList() pl.fromXMLFile(filename=pl_filename) for part in pl: remove(part.getFilename()) rmdir(pdir) remove(pl_filename)
def run(fname, outname, cores=6): even = ParticleList() even.fromXMLFile(fname) aa = averageParallel(particleList=even, averageName=outname, showProgressBar=True, verbose=False, createInfoVolumes=False, weighting=False, norm=False, setParticleNodesRatio=3, cores=cores)
def extractClassesFromPL(pl_name, class_names, output): from pytom.basic.structures import ParticleList pl = ParticleList() pl.fromXMLFile(pl_name) pls = pl.splitByClass() class_labels = class_names.split(',') res = ParticleList() for pp in pls: if pp[0].getClass() in class_labels: res += pp if output: res.toXMLFile(output) return res
def subTomoClust(particleListFilename, classifiedParticleListFilename, cccName, neig, nclass, verbose=False): """ subtomogram clustering using CCC and kmeans @param particleListFilename: particle list filename @type particleListFilename: str @param classifiedParticleListFilename: output particle list filename @type classifiedParticleListFilename: str @param cccName: Name of (Constrained) Correlation Coefficient (CCC) matrix @type cccName: str @param neig: number of eigenvectors @type neig: int @param nclass: number of classes @type nclass: int @author: FF Jan 2013 """ from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(particleListFilename) if verbose: print("Particle List read in") ccc = readCCC(cccName) if verbose: print("CCC read in") coeffs, eigvalues = SVD_analysis(ccc) if verbose: print("Eigen analysis done") labels = kmeansCluster(coeff=coeffs, neig=neig, nclass=nclass) if verbose: print("kmeans clustering done") for (ipart, part) in enumerate(pl): part.setClass(className=str(labels[ipart])) if verbose: print("Class labels assigned") pl.toXMLFile(classifiedParticleListFilename) if verbose: print("File written")
def get_size(particleList, directory): tempPL = ParticleList() tempPL.fromXMLFile(particleList) tomoName = tempPL[0].getPickPosition().getOriginFilename( ) if tempPL[0].getPickPosition().getOriginFilename( ) else tempPL[0].getSourceInfo().getTomoName() if not os.path.exists(tomoName): tomoName = os.path.join(directory, tomoName) if not os.path.exists(tomoName): return 'Failed' try: dimx, dimy, dimz = read_size(tomoName) except: print('Failed') return 'Failed' return [dimx, dimy, dimz]
def averageClasses(particleListFilename, avName): """ write class averages of classified particles @param particleListFilename: particle list filename @type particleListFilename: str @param avName: Name for class averages (<avName>_iclass.em) @type avName: str @author: FF @date: Jan 2013 """ from pytom.basic.structures import ParticleList pl = ParticleList() pl.fromXMLFile(particleListFilename) pl.sortByClassLabel() pls = pl.splitByClass() for cl in pls: className = cl[0].getClassName() cl.average(avName + "_" + str(className) + '.em') print(className, ' contains ', len(cl), ' particles')
def mirrorParticleList(particleList, outname, directory='./'): sizes = get_size(particleList, directory) if sizes == 'Failed': print( 'Mirroring particle coordinates did not succeed. Please ensure the paths to the origin tomogram are correct' ) return dimx, dimy, dimz = sizes tempPL = ParticleList() tempPL.fromXMLFile(particleList) for particle in tempPL: pp = particle.getPickPosition() pp.setX(dimx - pp.getX()) pp.setY(dimy - pp.getY()) pp.setZ(dimz - pp.getZ()) shift = particle.getShift() shift.invert() tempPL.toXMLFile(outname)
def readSetOfVolumes(volsXml, volSet, **kwargs): """ Populate a Scipion set of volumes from a given xml file from PyTom. """ # Add to the path the root to pytom backupPath = list(sys.path) addPyTomPaths() from pytom.basic.structures import Particle, ParticleList, Wedge from pytom.score.score import Score, PeakPrior from pytom.frm.FRMAlignment import FRMScore pl = ParticleList() pl.fromXMLFile(volsXml) xmlDir = os.path.dirname(volsXml) for particle in pl: volFn = os.path.join(xmlDir, particle.getFilename()) vol = em.Volume() vol.setFileName(volFn) vol.pytomInfo = readPyTomInfo(particle) volSet.append(vol) sys.path = backupPath
ScriptOption('-p', 'Particle list.', True, False), ScriptOption('-c', 'Class label file.', True, False), ScriptOption('-o', 'Output particle list.', True, False), ScriptOption('-t', 'True positive class.', True, True), ScriptOption(['-h', '--help'], 'Help.', False, True) ]) if len(sys.argv) == 1: print(helper) sys.exit() try: pl_filename, class_label_filename, output, tp_label, help = parse_script_options( sys.argv[1:], helper) except Exception as e: print(e) sys.exit() if help is True: print(helper) sys.exit() if tp_label is None: tp_label = 1 else: tp_label = int(tp_label) from pytom.basic.structures import ParticleList pl = ParticleList(".") pl.fromXMLFile(pl_filename) pl.setClassFromLocalizationClassList(class_label_filename) new_pl = pl.particlesFromClass(tp_label) new_pl.toXMLFile(output)
def run(self, verbose=False): from pytom_volume import read, sum from pytom.basic.filter import lowpassFilter from pytom.basic.correlation import nxcc from pytom.basic.structures import Rotation from pytom.tools.ProgressBar import FixedProgBar while True: # get the job job = self.get_job() try: pairs = job["Pairs"] pl_filename = job["ParticleList"] except: if verbose: print(self.node_name + ': end') break # get some non-job message, break it from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(pl_filename) if verbose: prog = FixedProgBar(0, len(pairs)-1, self.node_name+':') i = 0 # run the job result = {} last_filename = None binning = int(job["Binning"]) mask = read(job["Mask"], 0, 0, 0, 0, 0, 0, 0, 0, 0, binning, binning, binning) for pair in pairs: if verbose: prog.update(i) i += 1 g = pl[pair[0]] f = pl[pair[1]] vf = f.getTransformedVolume(binning) wf = f.getWedge().getWedgeObject() wf_rotation = f.getRotation().invert() # wf.setRotation(Rotation(-rotation[1],-rotation[0],-rotation[2])) # wf_vol = wf.returnWedgeVolume(vf.sizeX(), vf.sizeY(), vf.sizeZ(), True, -rotation[1],-rotation[0],-rotation[2]) vf = lowpassFilter(vf, job["Frequency"], 0)[0] if g.getFilename() != last_filename: vg = g.getTransformedVolume(binning) wg = g.getWedge().getWedgeObject() wg_rotation = g.getRotation().invert() # wg.setRotation(Rotation(-rotation[1],-rotation[0],-rotation[2])) # wg_vol = wg.returnWedgeVolume(vg.sizeX(), vg.sizeY(), vg.sizeZ(), True, -rotation[1],-rotation[0],-rotation[2]) vg = lowpassFilter(vg, job["Frequency"], 0)[0] last_filename = g.getFilename() score = nxcc( wg.apply(vf, wg_rotation), wf.apply(vg, wf_rotation), mask) # overlapped_wedge_vol = wf_vol * wg_vol # scaling = float(overlapped_wedge_vol.numelem())/sum(overlapped_wedge_vol) # score *= scaling result[pair] = score # send back the result self.send_result(result) pytom_mpi.finalise()
def polish_particles(particle_list_filename, projection_directory, averaged_subtomogram, binning, offset, projections, tilt_angles, mpi, fsc_path='', peak_border=75, outputDirectory='./', create_graphics=False, number_of_particles=0, verbose=False): """ To polish a particle list based on (an) initial subtomogram(s). :param particle_list_filename: the filename of the particlelist :type particle_list_filename: str :param projection_directory: the directory of the projections :type projection_directory: str :param averaged_subtomogram: to give a path to an averaged subtomogram to be used instead of subtomograms of all particles separately :type averaged_subtomogram: str :param binning: the binning factor used :type binning: int :param offset: the offset used (x, y, z) :type offset: list(int, int, int) :param projections: a list with filenames of projections :type projections: list(str) :param tilt_angles: the list of tiltangles used :type tilt_angles: list(int) :param mpi: the instance of mpi which can be used for multi threading :type mpi: mpi instance :param create_graphics: to create plots of major parts of the algorithm, mainly used for debugging and initial creation :type create_graphics: bool :param number_of_particles: to use a subset of the particles for the particle polishing :type number_of_particles: int :param skip_alignment: skips the alignment phase, does not do particle polishing :type skip_alignment: bool :return: nothing, it writes everything to disk :returntype: void """ assert number_of_particles == -1 or number_of_particles > 0 assert binning > 0 assert vol_size > 0 assert vol_size % 2 == 0 assert isinstance(projections, list) assert isinstance(vol_size, int) assert isinstance(binning, int) assert isinstance(offset, list) and len(offset) == 3 assert isinstance(offset[0], int) and isinstance( offset[1], int) and isinstance(offset[2], int) assert isinstance(tilt_angles, list) assert isinstance(particle_list_filename, str) assert isinstance(projection_directory, str) assert isinstance(create_graphics, bool) assert isinstance(averaged_subtomogram, str) assert isinstance(number_of_particles, int) assert isinstance(skip_alignment, bool) import numpy as np import os from pytom.tompy.io import read_size, read from pytom.basic.datatypes import fmtLAR, headerLocalAlignmentResults, LOCAL_ALIGNMENT_RESULTS # load particle list from pytom.basic.structures import ParticleList particlelist = ParticleList() particlelist.fromXMLFile(particle_list_filename) particle_list_name = os.path.splitext( os.path.basename(str(particle_list_filename)))[0] if number_of_particles > 0: particlelist = particlelist[:number_of_particles] if verbose: print(len(particlelist)) print("{:s}> Creating the input array".format(gettime())) dimz = read_size(particlelist[0].getPickPosition().getOriginFilename(), 'z') * binning vol_size = read_size(particlelist[0].getFilename(), 'x') input_to_processes = [] # data = {} # for projectioname in projections: # data[projectioname] = read(projectioname) # # template = read(averaged_subtomogram) input_to_processes = [] for particle_number, particle in enumerate(particlelist): rot = (particle.getRotation().getZ1(), particle.getRotation().getX(), particle.getRotation().getZ2()) # loop over tiltrange, take patch and cross correlate with reprojected subtomogram for img, ang in zip(projections, tilt_angles): input_to_processes.append([ averaged_subtomogram, ang, offset, vol_size, particle.getPickPosition().toVector(), rot, particle.getFilename(), particle_number, binning, img, create_graphics, fsc_path, dimz, peak_border, ]) if verbose: print(len(input_to_processes)) print("{:s}> Created the input array".format(gettime())) if verbose: print("{:s}> Started on running the process".format(gettime())) lists = list(zip(*input_to_processes)) if verbose: print(len(list(lists))) output = mpi.parfor( run_single_tilt_angle, list( zip(lists[0], lists[1], lists[2], lists[3], lists[4], lists[5], lists[6], lists[7], lists[8], lists[9], lists[10], lists[11], lists[12], lists[13]))) # Some problem internally 23/10/2019 results_file = os.path.join(outputDirectory, f"resultsPolish_{particle_list_name}.txt") np.savetxt(results_file, np.array(output, dtype=LOCAL_ALIGNMENT_RESULTS), fmt=fmtLAR, header=headerLocalAlignmentResults) if verbose: print("{:s}> Ran the processes".format(gettime()))
from pytom.tools.parse_script_options import parse_script_options from pytom.basic.structures import ParticleList, SingleTiltWedge helper = ScriptHelper(sys.argv[0].split('/')[-1], # script name description='Set the same wedge to all the particles in the particle list.', authors='Yuxiang Chen', options=[ScriptOption(['-p'], 'Particle list', True, False), ScriptOption(['-w'], 'Wedge Angle (degree)', True, False), ScriptOption(['-o'], 'Output particle list', True, True), ScriptOption(['-h', '--help'], 'Help.', False, True)]) if len(sys.argv) == 1: print(helper) sys.exit() try: pl_name, wedge_angle, output, bHelp = parse_script_options(sys.argv[1:], helper) except: sys.exit() if bHelp is True: print(helper) sys.exit() pl = ParticleList() pl.fromXMLFile(pl_name) w = SingleTiltWedge(int(wedge_angle)) pl.setWedgeAllParticles(w) if output is None: pl.toXMLFile(pl_name) else: pl.toXMLFile(output)
from pytom_volume import read from sh.frm import frm_align_vol from pytom.tools.maths import rotation_distance, euclidianDistance from pytom.tools.timing import Timing from pytom.basic.structures import ParticleList from pytom.basic.structures import Shift, Rotation pl = ParticleList('.') pl.fromXMLFile('/fs/home/ychen/4Chen/first100.xml') r = read('/fs/home/ychen/4Chen/avg_first100.em') for pp in pl: v = read(pp.getFilename()) pos, ang = frm_align_vol(v, [-60.0, 60.0], r, [8, 32], 10, mask=30) pp.setShift( Shift([ pos[0] - v.sizeX() / 2, pos[1] - v.sizeY() / 2, pos[2] - v.sizeZ() / 2 ])) pp.setRotation(Rotation(ang)) pl.average('average.em', True)
##!/bin/bash ## This script merges desired classes from one or multiple particleLists of individual tomograms. It utilizes the pytom function pl.splitByClass ## Utrecht University, 19.11.2019, RE #module load openmpi/2.1.1 pytom/0.971 from pytom.basic.structures import ParticleList import os ## The variable nice_classes_list contains arrays specifying the tomogram and classes. Each array should contain the index of the desired tomograms in the first position, followed by the indices of the desired classes from that tomogram. ## In the example below, the first array [8,0,1,3,4] specifies the classes 0, 1, 3 and 4 within tomogram 8. nice_classes_list = [[8,0,1,3,4], [9,0], [13,2], [14,0,1,2], [15,0,1,3], [16,0,1], [20,1], [21,3], [24,1], [27,2], [29,1], [30,1], [31,0], [32,1], [33,1], [34,1], [35,1], [36,1,4], [38,1], [43,3], [45,1], [46,1], [52,4], [53,1,3], [54,1], [79,3]] pl=ParticleList() ## Below, adjust the path inside pl.fromXMLFile to point to the individual particle lists. for nice_classes in nice_classes_list: pl.fromXMLFile('tomogram' + str(nice_classes_list[nice_classes][0]) + '/subtomograms/subtomos_bin6/AC3D/classified_pl_iter9.xml') lists=pl.splitByClass() for i in range(len(nice_classes_list[nice_classes])-1) ribos=ribos.append(lists[nice_classes_list[nice_classes][i + 1]) ## specify the desired output path below ribos.toXMLFile('AC3d_pickedClasses.xml')
def updatePL(fnames, outnames, directory='', suffix='', wedgeangles=[], multiplypickpos=1, multiplyshift=None, new_center=[], sizeSubtomo=64, move_shift=False, binSubtomo=1, binRecon=1, rotation=[], anglelist='', mirror=False, tomogram_dir='./', convention='zxz'): if type(fnames) == str: fnames = [fnames] if type(outnames) == str: outnames = [outnames] try: wedgelen = len(wedgeangles) except: wedgelen = 0 for n, xmlfile in enumerate(fnames): tempPL = ParticleList() tempPL.fromXMLFile(xmlfile) for particle in tempPL: # Update directory to particle if directory: filename = os.path.join( directory, os.path.basename(particle.getFilename())) particle.setFilename(filename) # Add suffix to directory name in which particle is stored # e.g. suffix = _bin3 # --> Subtomograms/tomogram_000/particle_1.em will become Subtomograms/tomogram_000_bin3/particle_1.em if suffix: filename = particle.getFilename() filename = os.path.join( os.path.dirname(filename) + suffix, os.path.basename(filename)) particle.setFilename(filename) # Update wedge angles of angle1 and angle2 if wedgelen > n + 1: w = particle.getWedge() w.setWedgeAngles(wedgeangles[n * 2:n * 2 + 2]) # Multiply pick position if abs(multiplypickpos - 1) > 1E-3: pp = particle.getPickPosition() pp.scale(multiplypickpos) # Shift is multiply by the respective binning factor. if not (multiplyshift is None): shift = particle.getShift() shift.scale(multiplyshift) # Randomize the angles of all particles in particle list. if type(anglelist) == type(numpy.array([])): cc = 180. / numpy.pi import random z1, z2, x = random.choice(anglelist) particle.setRotation(rotation=Rotation( z1=z1 * cc, z2=z2 * cc, x=x * cc, paradigm='ZXZ')) shift = particle.getShift() angles = particle.getRotation().toVector(convention=convention) rot_particleList = R.from_euler(convention, angles, degrees=True) if new_center: new_center_vector = numpy.array(new_center) - sizeSubtomo // 2 new_center_vector_rotated = rot_particleList.apply( new_center_vector) shift.addVector(new_center_vector_rotated) if move_shift == True: pp = particle.getPickPosition() shift.scale(binSubtomo / binRecon) ss = shift.toVector() pp.setX(pp.getX() + ss[0]) pp.setY(pp.getY() + ss[1]) pp.setZ(pp.getZ() + ss[2]) particle.setPickPosition(pp) shift.scale(0.) #print(particle) # Combine rotations from particleList and rotation if rotation: rot_rotation = R.from_euler(convention, rotation, degrees=True) combined_rotation = rot_particleList * rot_rotation z1, x, z2 = combined_rotation.as_euler(convention, degrees=True) particle.setRotation( rotation=Rotation(z1=z1, z2=z2, x=x, paradigm='ZXZ')) if mirror: # Update shifts shift.scale(-1) particle.setShift(shift) # Update angles as well rotationT = particle.getRotation() rotationT.setZ1(-1 * rotationT.getZ1()) rotationT.setZ2(-1 * rotationT.getZ2()) rotationT.setX(-1 * rotationT.getX()) particle.setRotation(rotationT) tempPL.toXMLFile(outnames[n]) if mirror: mirrorParticleList(outnames[n], outnames[n], directory=tomogram_dir)
print('Command not right. Exit!') sys.exit() for o, a in opts: if o in ("-h", "--help"): print(usage) sys.exit() if o in ("-l"): projDir = a if o in ("-p"): plFilename = a if o in ("-b"): binning = int(a) if o in ("-s"): size = int(a) if o in ("-f"): offset = [int(i) for i in a.split(",")] from pytom.basic.structures import ParticleList pl = ParticleList('.') pl.fromXMLFile(plFilename) pl.setCenterInPickVolume( -offset[0], -offset[1], -offset[2]) # thomas has changed the sign of it!!! from pytom.reconstruction.reconstructionStructures import ProjectionList projs = ProjectionList() projs.loadDirectory(projDir) projs.generateVolumes(pl, size, binning, False, True, False)
def interpretRequestParameters(parameters): """ interpretRequestParameters """ from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge from pytom.alignment.preprocessing import Preprocessing from pytom.frontend.serverpages.serverMessages import FileMessage particleList = ParticleList() if 'plXML' in parameters: particleList.fromXMLFile(parameters['plXML']) elif 'plDIR' in parameters: particleList = ParticleList(parameters['plDIR']) particleList.loadDirectory() else: raise RuntimeError('ParticleList parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(parameters['pixSize']) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(parameters['partDia']) else: raise RuntimeError('Particle diameter missing in request!') if 'wa1' in parameters: wedgeAngle1 = float(parameters['wa1']) else: raise RuntimeError('Wedge angle 1 parameter missing in request!') if 'wa2' in parameters: wedgeAngle2 = float(parameters['wa2']) else: raise RuntimeError('Wedge angle 2 parameter missing in request!') wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2]) if 'mask' in parameters: mask = Mask(parameters['mask']) else: raise RuntimeError('Mask parameter missing in request!') if not 'lowestF' in parameters: raise RuntimeError('Lowest frequency parameter missing in request!') if not 'highestF' in parameters: raise RuntimeError('Highest frequency parameter missing in request!') if not 'filtSm' in parameters: raise RuntimeError('Filter smooth parameter missing in request!') preprocessing = Preprocessing(float(parameters['lowestF']), float(parameters['highestF']), float(parameters['filtSm'])) score = None if 'score' in parameters: if parameters['score'] == 'xcf': from pytom.score.score import xcfScore as scoreClass elif parameters['score'] == 'nxcf': from pytom.score.score import nxcfScore as scoreClass elif parameters['score'] == 'flcf': from pytom.score.score import FLCFScore as scoreClass score = scoreClass() else: raise RuntimeError('Score parameter missing in request!') if 'iter' in parameters: iterations = int(parameters['iter']) else: raise RuntimeError('Number of iterations missing in request!') if 'binning' in parameters: binning = int(parameters['binning']) else: raise RuntimeError('Scaling parameter missing in request!') if 'classes' in parameters: numberClasses = float(parameters['classes']) else: raise RuntimeError('Number classes parameter missing in request!') if 'conv' in parameters: convergence = float(parameters['conv']) else: raise RuntimeError('Convergence parameter missing in request!') if 'dest' in parameters: destination = parameters['dest'] else: raise RuntimeError('Destination parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(float(parameters['pixSize'])) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(float(parameters['partDia'])) else: raise RuntimeError('Particle diameter missing in request!') from pytom.cluster.mcoEXMXStructures import MCOEXMXJob job = MCOEXMXJob(particleList, iterations, destination, mask, score, preprocessing, wedgeInfo, binning, sampleInfo, numberClasses, convergence, symmetry=None) jobXMLFile = '' if 'jobFile' in parameters: jobXMLFile = parameters['jobFile'] job.toXMLFile(jobXMLFile) jobRunFile = jobXMLFile[0:-3] createRunscripts(jobRunFile + 'sh', jobXMLFile) return FileMessage('MCOEXMXJob', jobXMLFile, 'created')
def extractParticleListsClosestToRefMarker(xmlfile, markerfile, binning_factor=8, directory='./', projDirTemplate=''): from pytom.basic.structures import PickPosition, ParticleList pL = ParticleList() pL.fromXMLFile(os.path.join(directory, xmlfile)) dict_particle_lists = {} for particle in pL: tomogram = particle.getPickPosition().getOriginFilename().split( '/')[-1].split('.')[0] if not tomogram: tomogram = particle.getSourceInfo().getTomoName().split('.')[0] if not tomogram in dict_particle_lists.keys(): dict_particle_lists[tomogram] = ParticleList() dict_particle_lists[tomogram].append(particle) try: markers = loadstar(markerfile, dtype=datatypeMR) except: correct_header_markerfile(markerfile) markers = loadstar(markerfile, dtype=datatypeMR) xmlsCM = [] for pl_key in dict_particle_lists.keys(): outLists = {} for particle in dict_particle_lists[pl_key]: x, y, z = particle.getPickPosition().toVector() x *= binning_factor y *= binning_factor z *= binning_factor closestMarkerIndex = determine_closest_marker(x, y, z, markers) projectionDirectory = projDirTemplate.replace( '_CLOSEST_', '_{:04d}_'.format(closestMarkerIndex)) markerPositionFile = f'{projectionDirectory}/markerLocations_irefmark_{closestMarkerIndex}.txt' realignmarkers = loadstar(markerPositionFile, dtype=datatypeMR) if not closestMarkerIndex in outLists.keys(): outLists[closestMarkerIndex] = ParticleList() ox, oy = determineShiftXY(markers, realignmarkers) oz = markers['OffsetZ'][closestMarkerIndex] originFname = particle.getPickPosition().getOriginFilename() print(x, y, z, ox, oy, oz) pp = PickPosition(x=(x - ox) / binning_factor, y=(y - oy) / binning_factor, z=((z - oz) / binning_factor), originFilename=originFname) particle.setPickPosition(pp) outLists[closestMarkerIndex].append(particle) for markerIndex in outLists.keys(): outfname = '.tempCM_particleList_{}_refMarkerIndex_{}.xml'.format( pl_key, markerIndex) outfname = os.path.join(directory, outfname) outLists[markerIndex].toXMLFile(outfname) xmlsCM.append([markerIndex, outfname]) return xmlsCM
def run(parameters, verbose=False): """ run: Answers a http request depending on the provided parameters. Parameters can be XML=FILENAME DIR=DIRNAME ALIG=NAME @param parameters: """ from pytom.basic.structures import ParticleList from pytom.tools.files import checkDirExists, checkFileExists if verbose: print("Parsing particleList request!") splitParameters = parameters.split('&') if splitParameters.__class__ == list: if verbose: print(splitParameters) for i in range(len(splitParameters)): parameter = splitParameters[i] split = parameter.split('=') keyword = split[0] argument = split[1] if verbose: print('Keyword : ', keyword) print('Arguments : ', argument) if keyword == 'XML': from pytom.tools.files import readStringFile, getPytomPath import io from lxml import etree if not checkFileExists(argument): raise IOError('File not found!') pl = ParticleList('/') pl.fromXMLFile(argument) xsltString = readStringFile( getPytomPath() + '/frontend/html/xslt/ParticleList.xsl') xsltTransform = io.StringIO(xsltString) transformed = pl.xsltTransform(xsltTransform) return etree.tostring(transformed, pretty_print=True) elif keyword == 'DIR': from pytom.tools.files import checkDirExists, getPytomPath, readStringFile import io from lxml import etree if not checkDirExists(argument): raise IOError('File not found!') pl = ParticleList(argument) pl.loadDirectory() xsltString = readStringFile( getPytomPath() + '/frontend/html/xslt/ParticleList.xsl') xsltTransform = io.StringIO(xsltString) transformed = pl.xsltTransform(xsltTransform) return etree.tostring(transformed, pretty_print=True) elif keyword == 'ALIG': from pytom.tools.files import checkDirExists, getPytomPath, readStringFile import io from lxml import etree if not checkDirExists(argument): raise IOError('File not found!') pl = ParticleList(argument) pl.fromAlignmentFile(argument) xsltString = readStringFile( getPytomPath() + '/frontend/html/xslt/ParticleList.xsl') xsltTransform = io.StringIO(xsltString) transformed = pl.xsltTransform(xsltTransform) return etree.tostring(transformed, pretty_print=True) elif splitParameters.__class__ == str: if verbose: print(splitParameters)
def test(index=0): from pytom.tompy.io import read from pytom.tompy.transform import rotate3d path_raw_projection = "/data2/dschulte/BachelorThesis/Data/VPP2/03_Tomographic_Reconstruction/tomogram_000/sorted/sorted_29.em" path_aligned_projection = "/data2/dschulte/BachelorThesis/Data/VPP2/03_Tomographic_Reconstruction/tomogram_000/alignment/marker_0001_-60.0,60.0/sorted_aligned_30.em" path_template = "/data2/dschulte/BachelorThesis/Data/VPP2/05_Subtomogram_Analysis/combo_reduced.em" tilt_angle = -1.9989999533 raw_projection = read(path_raw_projection) aligned_projection = read(path_aligned_projection) template = read(path_template) dim = aligned_projection.shape[0] from pytom.basic.structures import ParticleList particlelist1 = ParticleList() particlelist1.fromXMLFile( "/data2/dschulte/BachelorThesis/Data/VPP2/04_Particle_Picking/Picked_Particles/combined_reduced_extracted/particleList_combined_reduced_tomogram_010_WBP.xml" ) align_results = (-15.3044300079, 3.7495634556, -184.3835906982, 1.0000053644) # -2 #align_results = (-1.2395387888, 4.9647006989, -184.3754882812, 1.0000000000) # 0 #pick_position = (278.12318382089245 * 8, 222.6395540890773 * 8, 268.97256780848085 * 8) #0 #pick_position = (381.21906883806 * 8, 153.61353397521387 * 8, 246.8315433927568 * 8) #74 #particle_rotation = (26.442828473505173, 44.44149544840194, 58.160958298848676) #0 #particle_rotation = (85.2456894956599, 30.815061362336394, 9.543300915975514) #74 pick_position = particlelist1[index].getPickPosition().toVector() particle_rotation = (particlelist1[index].getRotation().getZ1(), particlelist1[index].getRotation().getX(), particlelist1[index].getRotation().getZ2()) print(pick_position, particle_rotation) align_transformation, raw_position, aligned_position, template_transformation = combine_trans_projection( align_results, pick_position, particle_rotation, tilt_angle, dim, 8, template.shape[0]) d = 100 raw_patch = raw_projection[int(raw_position[0] - d):int(raw_position[0] + d), int(raw_position[1] - d):int(raw_position[1] + d)].squeeze() raw_patch = raw_patch / np.mean(raw_patch) aligned_patch = aligned_projection[int(aligned_position[0] - d):int(aligned_position[0] + d), int(aligned_position[1] - d):int(aligned_position[1] + d)].squeeze() aligned_patch = aligned_patch / (np.mean(aligned_patch)) aligned_raw = matrix_apply_to_2d(aligned_projection.squeeze(), align_transformation) aligned_raw_patch = aligned_raw[int(raw_position[0] - d):int(raw_position[0] + d), int(raw_position[1] - d):int(raw_position[1] + d)].squeeze() aligned_raw_patch = aligned_raw_patch / (np.mean(aligned_raw_patch)) transformed_template = matrix_apply_to_3d_3x3(template, template_transformation) print(np.mean(transformed_template), np.mean(template)) template_2d = transformed_template.sum(axis=2) template_2d = template_2d / np.mean(template_2d) print(template_2d.shape) template_vol = rotate3d(template, phi=particle_rotation[0], the=particle_rotation[1], psi=particle_rotation[2]) template_vol = rotate3d(template_vol, the=tilt_angle) template_vol_2d = template_vol.sum(axis=2) from pytom.reconstruction.reconstruct_local_alignment import normalised_cross_correlation_numpy, find_sub_pixel_max_value_2d raw_cc = normalised_cross_correlation_numpy(raw_patch, template_2d) rx, ry, _ = find_sub_pixel_max_value_2d(raw_cc) aligned_cc = normalised_cross_correlation_numpy(aligned_patch, template_vol_2d) tx, ty, _ = find_sub_pixel_max_value_2d(aligned_cc) import pylab as pl import scipy f, ax = pl.subplots(2, 3, figsize=(15, 10)) for i in range(2): for j in range(3): ax[i][j].axis('off') ax[0][0].set_title('Raw Data Particle') ax[0][0].imshow(scipy.ndimage.gaussian_filter(raw_patch, 3)) ax[0][1].set_title('Template Transformed to Raw Data') ax[0][1].imshow(template_2d) ax[0][2].set_title('Cross Correlation') ax[0][2].imshow(raw_cc) ax[0][2].text(0.05, 0.05, 'Peak\nx: {0:.2f}\ny: {0:.2f}'.format(rx, ry), transform=ax[0][2].transAxes, color='white') ax[1][0].set_title('Aligned Data Particle') ax[1][0].imshow(scipy.ndimage.gaussian_filter(aligned_patch, 3)) ax[1][1].set_title('Template Aligned to Aligned Data') ax[1][1].imshow(template_vol_2d) ax[1][2].set_title('Cross Correlation') ax[1][2].imshow(aligned_cc) ax[1][2].text(0.05, 0.05, 'Peak\nx: {0:.2f}\ny: {0:.2f}'.format(tx, ty), transform=ax[1][2].transAxes, color='white') f.tight_layout() pl.show()
print("binning = ", str(settings["binning"])) print("mask = ", str(settings["mask"])) print("fmask = ", str(settings["fmask"])) print("niteration = ", str(settings["niteration"])) print("dispersion = ", str(settings["dispersion"])) print("threshold = ", str(settings["threshold"])) print("noise = ", str(settings["noise"])) print("noalign = ", str(settings["noalign"])) print("outputdir = ", str(settings["output_directory"])) print("noise = ", str(settings["noise"])) print('----------------------------------------------------') print() # start the clustering mpi.begin() import time tt = time.time() try: pl = ParticleList() pl.fromXMLFile(options.filename) classify(pl, settings) except Exception as e: print(e) finally: mpi.end() print(f'total time: {time.time()-tt} sec')
def fromXML(self, xmlObj): """ read from xml file @param xmlObj: xml object @type xmlObj: L{lxml.etree.Element} """ from lxml.etree import _Element if xmlObj.__class__ != _Element: raise Exception('You must provide a valid XML object.') if xmlObj.tag == "FRMJob": jobDescription = xmlObj else: jobDescription = xmlObj.xpath('FRMJob') if len(jobDescription) == 0: raise Exception("This XML is not a FRMJob.") jobDescription = jobDescription[0] from pytom.basic.structures import ParticleList, Reference, Mask, SampleInformation, MultiSymmetries pl = ParticleList('.') particleList_element = jobDescription.xpath('ParticleList') if len(particleList_element) > 0: pl.fromXML(particleList_element[0]) else: list_elements = jobDescription.xpath('ParticleListLocation') for e in list_elements: sub_pl = ParticleList() sub_pl.fromXMLFile(e.get('Path')) pl += sub_pl self.particleList = pl r = jobDescription.xpath('Reference')[0] self.reference = Reference('') self.reference.fromXML(r) m = jobDescription.xpath('Mask')[0] self.mask = Mask('') self.mask.fromXML(m) try: si = jobDescription.xpath('SampleInformation')[0] self.sampleInformation = SampleInformation() self.sampleInformation.fromXML(si) except: self.sampleInformation = SampleInformation() try: syms = jobDescription.xpath('MultiSymmetries')[0] self.symmetries = MultiSymmetries() self.symmetries.fromXML(syms) except: self.symmetries = MultiSymmetries() self.peak_offset = int(jobDescription.get('PeakOffset')) self.bw_range = [ int(i) for i in jobDescription.get('BandwidthRange')[1:-1].split(',') ] self.freq = int(jobDescription.get('Frequency')) self.destination = jobDescription.get('Destination') self.max_iter = int(jobDescription.get('MaxIterations')) self.r_score = jobDescription.get('RScore') == 'True' self.weighting = jobDescription.get('WeightedAverage') == 'True' self.bfactor = jobDescription.get('BFactor') self.binning = int(jobDescription.get('binning')) if jobDescription.get('AdaptiveResolution'): adaptive_resolution = jobDescription.get('AdaptiveResolution') if adaptive_resolution == '+1': self.adaptive_res = False # always increase by 1 else: self.adaptive_res = float(adaptive_resolution) else: self.adaptive_res = 0.0 # default, if not specified if jobDescription.get('FSC'): self.fsc_criterion = float(jobDescription.get('FSC')) else: self.fsc_criterion = 0.5 # default value # for the constraint try: from sh_alignment.constrained_frm import AngularConstraint con = jobDescription.xpath('AngularConstraint') if len(con) != 0: ac = AngularConstraint() c = ac.fromXML(con[0]) self.constraint = c else: self.constraint = None except: self.constraint = None
sys.argv[1:], helper) except Exception as e: print(e) sys.exit() if help is True: print(helper) sys.exit() try: cores = int(cores) except: cores = 1 if not os.path.exists(plName): print('Please provide an existing particle list') sys.exit() even = ParticleList() even.fromXMLFile(plName) averageParallel(particleList=even, averageName=outname, showProgressBar=showProgressBar, verbose=verbose, createInfoVolumes=createInfoVol, weighting=weighting, norm=norm, setParticleNodesRatio=3, cores=cores)
continue tmp = projections[0] sx = tmp.getXSize( ) # here should be the size of original projection! sy = tmp.getYSize() recOffset2 = [0, 0, 0] recOffset2[0] = -sx / 2 + recOffset[0] * coordinateBinning recOffset2[1] = -sy / 2 + recOffset[1] * coordinateBinning recOffset2[2] = -sx / 2 + recOffset[2] * coordinateBinning # set particle list in order to reconstruct subtomograms particleList = ParticleList() try: particleList.fromXMLFile(particleListXMLPath) except RuntimeError: print('Error reading particleList XML file! Abort') sys.exit() from pytom.basic.structures import PickPosition for particle in particleList: pickPosition = particle.getPickPosition() x = (pickPosition.getX() * coordinateBinning + recOffset2[0]) y = (pickPosition.getY() * coordinateBinning + recOffset2[1]) z = (pickPosition.getZ() * coordinateBinning + recOffset2[2]) particle.setPickPosition(PickPosition(x=x, y=y, z=z)) if alignResultFile: print(f'Alignment result file is used.\n\n{alignResultFile}')
]) if len(sys.argv) == 1: print(helper) sys.exit() try: particleListName, volumeName, result, numberSymmetries, z1, x, z2, help = parse_script_options( sys.argv[1:], helper) except Exception as e: print(e) sys.exit() if particleListName: pl = ParticleList('.') pl.fromXMLFile(particleListName) elif volumeName: volume = read(volumeName) else: raise RuntimeError( 'You must specify either a particle list or a volume file for symmetrization.' ) if not z2: z2 = 0 if not x: x = 0 symmetry = PointSymmetry(numberSymmetries, z2, x) if particleListName:
if not checkFileExists(mask): raise RuntimeError('Mask file ' + mask + ' does not exist!') if not checkDirExists(destination): raise RuntimeError('Destination directory ' + destination + ' does not exist!') from pytom.cluster.mcoEXMXStructures import MCOEXMXJob from pytom.basic.structures import ParticleList, Reference, Mask, Wedge, SampleInformation, PointSymmetry from pytom.score.score import FLCFScore from pytom.frontend.serverpages.createMCOEXMXJob import createRunscripts from pytom.alignment.preprocessing import Preprocessing p = ParticleList() p.fromXMLFile(particleList) m = Mask(mask) w = Wedge([float(wedge1), float(wedge2)]) pre = Preprocessing(lowestFrequency=float(lowestFrequency), highestFrequency=float(highestFrequency)) sample = SampleInformation(pixelSize=float(pixelSize), particleDiameter=float(diameter)) if symmetryN is None or symmetryAxisZ is None or symmetryAxisX is None: sym = None else: sym = PointSymmetry(nfold=int(symmetryN), z2=float(symmetryAxisZ), x=float(symmetryAxisX)) job = MCOEXMXJob(particleList=p,numberIterations=numberIterations,\
print(helper) sys.exit() try: volFilename, plFilename, cubeSize, help = parse_script_options( sys.argv[1:], helper) except: sys.exit() if help is True: print(helper) sys.exit() cubeSize = int(cubeSize) particleList = ParticleList() try: particleList.fromXMLFile(plFilename) except: from pytom.localization.structures import readParticleFile particles = readParticleFile(plFilename) particleList = ParticleList() for particle in particles: particleList.append(particle.toParticle()) particlePath = particleList[0].getFilename() particleFolder = particlePath[0:particlePath.rfind('/')] if not checkDirExists(particleFolder): os.makedirs(particleFolder) prog = FixedProgBar(0, len(particleList) - 1, '')