def _storeUcscInfoBox(self, genome, parameterForm, trackName): htmlText = mechanize.urlopen(parameterForm.click('hgta_doSchema')).read() htmlText = htmlText.replace('h2>','H2>').replace('p>', 'P>').replace('<a href','<A HREF') #open('/tmp/TrackInfo_UcscHtmlDump.txt','w').write(htmlText) htmlSubText = htmlText[htmlText.find('<H2>'):htmlText.rfind('</P>')].replace('<A HREF="http:..', ' <A HREF="http://genome.ucsc.edu') #logMessage('htmlTrackInfo: '+htmlSubText[:50]) if htmlSubText: htmlSubText += '</P>' print ' : '.join(trackName) +', '+genome trackInfoObject = TrackInfo(genome, trackName) tInfoVarConverter = {'Description':'description', 'Methods':'methods', 'Credits':'credits','References':'reference', 'Data Release Policy':'restrictions', 'Display Conventions and Configuration':'displayConvConf'} htmlSubList = htmlSubText.split('<H2>')[1:] for i in htmlSubList: header = i[:i.find('</H2>')].replace(':','').strip() if header in tInfoVarConverter.keys(): setattr(trackInfoObject, tInfoVarConverter[header], i.split('</H2>')[1]) #if header == 'Description': # print 'Description' # trackInfoObject.description = i.split('</H2>')[1] #elif header == 'Methods': # print 'Methods' # trackInfoObject.methods = i.split('</H2>')[1] #elif header == 'Credits': # print 'Credits' # trackInfoObject.credits = i.split('</H2>')[1] #elif header == 'References': # print 'References' # trackInfoObject.reference = i.split('</H2>')[1] #else: # pass trackInfoObject.store()
def _trackFormatsAreEqual(genome, trackName1, trackName2): trackInfoHash1 = TrackInfo(genome, trackName1).trackFormatHash if trackInfoHash1: trackInfoHash2 = TrackInfo(genome, trackName2).trackFormatHash if trackInfoHash1 == trackInfoHash2: return True return False
def removeOutdatedPreProcessedFiles(cls, genome, trackName, allowOverlaps, mode): collector = PreProcMetaDataCollector(genome, trackName) if cls.preProcFilesExist(genome, trackName, allowOverlaps) and not \ collector.hasRemovedPreProcFiles(allowOverlaps): dirPath = createDirPath(trackName, genome, allowOverlaps=allowOverlaps) assert (dirPath.startswith(PROCESSED_DATA_PATH)) if mode == 'Real': print 'Removing outdated preprocessed data: ', dirPath for fn in os.listdir(dirPath): fullFn = os.path.join(dirPath, fn) if os.path.isfile(fullFn): os.unlink(fullFn) if os.path.isdir(fullFn): if cls._isOldTypeChromDirectory(fullFn, genome): shutil.rmtree(fullFn) else: print 'Would now have removed outdated preprocessed data if real run: ', dirPath collector.updateRemovedPreProcFilesFlag(allowOverlaps, True) if mode == 'Real': ti = TrackInfo(genome, trackName) ti.resetTimeOfPreProcessing()
def renameExistingStdTrackIfNeeded(cls, genome, stdTrackName): oldTrackName = None for allowOverlaps in [False, True]: parentDir = createDirPath(stdTrackName[:-1], genome, allowOverlaps=allowOverlaps) if os.path.exists(parentDir): dirContents = os.listdir(parentDir) realDirs = [ x for x in dirContents if os.path.isdir(os.path.join(parentDir, x)) and not os.path.islink(os.path.join(parentDir, x)) ] reqDirName = stdTrackName[-1] reqDirPath = os.path.join(parentDir, reqDirName) from gold.application.LogSetup import logMessage logMessage('Checking ' + reqDirPath) if os.path.islink(reqDirPath) and not os.path.isdir( os.readlink(reqDirPath)): # This is to fix a bug that ended in the symlink pointing to a file os.remove(reqDirPath) logMessage('Removed ' + reqDirPath) if realDirs and reqDirName not in dirContents: oldTrackName = stdTrackName[:-1] + [realDirs[0]] os.symlink(realDirs[0], reqDirPath) if oldTrackName is not None: ti = TrackInfo(genome, oldTrackName) ti.trackName = stdTrackName ti.store()
def handleTask(self, task): task.cls.createTrackChr(task.genome, task.inTrackName, task.outTrackName, task.windowSize, task.func, task.chr) #todo: rewrite to new preprocessor solution... TrackInfo.finalizeTrack(task.genome, task.outTrackName, ['val'], 'float64', 1)
def fixTrackInfo(trackName, genome='hg18'): """trackName [genome]""" "fix timeOfPreProcessing of TrackInfo for a given trackName, in order for it to become a valid selection.." trackName = trackName.split(':') ti = TrackInfo(genome, trackName) ti.timeOfPreProcessing = 'manualOverride' ti.store()
def removeOutdatedPreProcessedFiles(trackName, geSource, allowOverlaps, mode): genome = geSource.getGenome() if PreProcessUtils.preProcFilesExist(trackName, geSource, allowOverlaps) and not \ TrackInfoDataCollector(genome, trackName).hasRemovedPreProcFiles(allowOverlaps): dirPath = createDirPath(trackName, genome, allowOverlaps=allowOverlaps) assert( dirPath.startswith(PROCESSED_DATA_PATH) ) if mode == 'Real': print 'Removing outdated preprocessed data: ', dirPath for fn in os.listdir(dirPath): fullFn = os.path.join(dirPath, fn) if os.path.isfile(fullFn): os.unlink(fullFn) if os.path.isdir(fullFn): if not PreProcessUtils._isSubTrackDirectory(fullFn): shutil.rmtree(fullFn) else: print 'Would now have removed outdated preprocessed data if real run: ', dirPath TrackInfoDataCollector(genome, trackName).updateRemovedPreProcFilesFlag(allowOverlaps, True) if mode == 'Real': ti = TrackInfo(genome, trackName) ti.resetTimeOfPreProcessing()
def writemetainfo(bigdict, genome, startname, ucscname): print "inne i createdirs" for key in bigdict: thisdict = bigdict[key] thistrack = list(startname) thistrack.append(thisdict['antibody']) cellname = thisdict['cell'] if 'replicate' in thisdict: cellname = cellname + '_rep' + thisdict['replicate'] thistrack.append(cellname) print thistrack ti = TrackInfo(genome, thistrack) ti.description = 'TF bindingsites as reported in Peak-files Fetched from UCSC Genome browser, se http://genome.ucsc.edu/cgi-bin/hgTrackUi?db=hg19&g=' + ucscname ti.description = ti.description + '<BR/>' + 'Use of data might be restricted, see dateUnrestricted below.<BR/> ' for key in thisdict: ti.description = ti.description + '<BR/>' + key + '=' + thisdict[ key] ti.description = ti.description + '<BR/>Annotation from the "files.txt"' ti.reference = 'Downloaded from UCSC hgdownload.cse.ucsc.edu, goldenPath/hg19/encodeDCC/wgEncodeHaibTfbs, August 2012. <BR/>The full data policy is available at http://genome.ucsc.edu/ENCODE/terms.html.' ti.hbContact = '*****@*****.**' #ti.quality = x[3] print "ti=", ti ti.store()
def _storeUcscInfoBox(self, genome, parameterForm, trackName): htmlText = mechanize.urlopen(parameterForm.click('hgta_doSchema')).read() htmlText = htmlText.replace('h2>','H2>').replace('p>', 'P>').replace('<a href','<A HREF') #open('/tmp/TrackInfo_UcscHtmlDump.txt','w').write(htmlText) htmlSubText = htmlText[htmlText.find('<H2>'):htmlText.rfind('</P>')].replace('<A HREF="http:..', ' <A HREF="http://genome.ucsc.edu') #logMessage('htmlTrackInfo: '+htmlSubText[:50]) if htmlSubText: htmlSubText += '</P>' print ' : '.join(trackName) +', '+genome trackInfoObject = TrackInfo(genome, trackName) tInfoVarConverter = {'Description':'description', 'Methods':'methods', 'Credits':'credits','References':'reference', 'Data Release Policy':'restrictions', 'Display Conventions and Configuration':'displayConvConf'} htmlSubList = htmlSubText.split('<H2>')[1:] for i in htmlSubList: header = i[:i.find('</H2>')].replace(':','').strip() if header in tInfoVarConverter.keys(): setattr(trackInfoObject, tInfoVarConverter[header], i.split('</H2>')[1]) #if header == 'Description': # print 'Description' # trackInfoObject.description = i.split('</H2>')[1] #elif header == 'Methods': # print 'Methods' # trackInfoObject.methods = i.split('</H2>')[1] #elif header == 'Credits': # print 'Credits' # trackInfoObject.credits = i.split('</H2>')[1] #elif header == 'References': # print 'References' # trackInfoObject.reference = i.split('</H2>')[1] #else: # pass trackInfoObject.store()
def _removePreprocessedTrackData(self, trackName): self._removeDir( createDirPath(trackName, self.GENOME, allowOverlaps=False), trackName) self._removeDir( createDirPath(trackName, self.GENOME, allowOverlaps=True), trackName) TrackInfo.removeFilteredEntriesFromShelve(self.GENOME, trackName)
def _calcAndStoreSubTrackCount(self, trackName): ti = TrackInfo(self._genome, trackName) trackCount = 0 for subTrackName in ProcTrackOptions.getSubtypes(self._genome, trackName, True): subTrackCount = TrackInfo(self._genome, trackName + [subTrackName]).subTrackCount if subTrackCount: trackCount += subTrackCount if ti.isValid(): trackCount += 1 ti.subTrackCount = trackCount ti.store()
def modifyTnRecord(genome, oldTn, newTn, verbose): trackInfo = TrackInfo(genome, oldTn) assert trackInfo.trackName == oldTn assert trackInfo.timeOfPreProcessing is not None, 'ERROR: trackInfo-object not complete for TN (is this track preprocessed?): ' + str(oldTn) #if trackInfo.timeOfPreProcessing is None: #print 'WARNING: timeOfPreProcessing is None for: ',oldTn trackInfo.trackName = newTn if not ONLY_SIMULATION: trackInfo.store() if verbose: print '(Storing track-info with new tn: %s)' % str(newTn) else: if verbose: print 'Would now store track-info with new tn: %s' % str(newTn)
def modifyTnRecord(genome, oldTn, newTn, verbose): trackInfo = TrackInfo(genome, oldTn) assert trackInfo.trackName == oldTn assert trackInfo.timeOfPreProcessing is not None, 'ERROR: trackInfo-object not complete for TN (is this track preprocessed?): ' + str( oldTn) #if trackInfo.timeOfPreProcessing is None: #print 'WARNING: timeOfPreProcessing is None for: ',oldTn trackInfo.trackName = newTn if not ONLY_SIMULATION: trackInfo.store() if verbose: print '(Storing track-info with new tn: %s)' % str(newTn) else: if verbose: print 'Would now store track-info with new tn: %s' % str(newTn)
def _createPreprocessedGsuiteTrack(self, gSuiteTrack, baseFileName, doEncodeId, urlPrefix=None): from quick.application.ExternalTrackManager import ExternalTrackManager as ETM from gold.description.TrackInfo import TrackInfo self.genericVisit(gSuiteTrack) galaxyTN = ETM.constructGalaxyTnFromSuitedFn( gSuiteTrack.path, fileEnding=gSuiteTrack.suffix, name=baseFileName) trackName = ETM.getPreProcessedTrackFromGalaxyTN( gSuiteTrack.genome, galaxyTN, printErrors=False, printProgress=False, renameExistingTracksIfNeeded=False, doEncodeId=doEncodeId, urlPrefix=urlPrefix) trackType = TrackInfo(gSuiteTrack.genome, trackName).trackFormatName.lower() hbUri = HbGSuiteTrack.generateURI(trackName=trackName) return GSuiteTrack(hbUri, title=gSuiteTrack.title, trackType=trackType, genome=gSuiteTrack.genome, attributes=gSuiteTrack.attributes, comment=gSuiteTrack.comment)
def _createMemoItems(cls, stat, useTrackFormatAsKey=True): from gold.description.TrackInfo import TrackInfo genome = stat.getGenome() statId = getClassName(stat) + '_' + stat.VERSION configHash = stat.getConfigKey(ignoreUsername=False) trackHashList = [] tracks = [stat._track] if hasattr(stat, '_track2') and stat._track2 is not None: tracks.append(stat._track2) for track in tracks: if useTrackFormatAsKey and not cls._isRandomizedTrack(track): trackFormatHash = TrackInfo(genome, track.trackName).trackFormatHash if not trackFormatHash: raise MissingInfoError( 'Precalculated TrackFormat hash is missing from TrackInfo, ' 'for track "{}" of genome "{}". '.format( track.trackName, genome) + 'Please consider to repeat the preprocessing step.') trackHashList.append(trackFormatHash) else: trackHashList.append(track.getUniqueKey(genome)) return [stat._region, statId, configHash] + trackHashList
def visitGalaxyGSuiteTrack(self, gSuiteTrack): self.genericVisit(gSuiteTrack) from quick.application.ExternalTrackManager import ExternalTrackManager from gold.description.TrackInfo import TrackInfo if gSuiteTrack.hasExtraFileName(): baseFileName = os.path.basename(gSuiteTrack.uriWithoutSuffix) else: baseFileName = gSuiteTrack.title galaxyTN = ExternalTrackManager.constructGalaxyTnFromSuitedFn( gSuiteTrack.path, fileEnding=gSuiteTrack.suffix, name=baseFileName) trackName = ExternalTrackManager.getPreProcessedTrackFromGalaxyTN( gSuiteTrack.genome, galaxyTN, printErrors=False, printProgress=False, renameExistingTracksIfNeeded=False) trackType = TrackInfo(gSuiteTrack.genome, trackName).trackFormatName.lower() hbUri = HbGSuiteTrack.generateURI(trackName=trackName) return GSuiteTrack(hbUri, title=gSuiteTrack.title, trackType=trackType, genome=gSuiteTrack.genome, attributes=gSuiteTrack.attributes, comment=gSuiteTrack.comment)
def renameExistingStdTrackIfNeeded(cls, genome, stdTrackName): oldTrackName = None for allowOverlaps in [False, True]: parentDir = createDirPath(stdTrackName[:-1], genome, allowOverlaps=allowOverlaps) if os.path.exists(parentDir): dirContents = os.listdir(parentDir) if len(dirContents) == 1 and dirContents[0] != stdTrackName[-1]: oldDir = parentDir + os.sep + dirContents[0] oldTrackName = stdTrackName[:-1] + [dirContents[0]] newDir = parentDir + os.sep + stdTrackName[-1] os.rename(oldDir, newDir) if oldTrackName is not None: ti = TrackInfo(genome, oldTrackName) ti.trackName = stdTrackName ti.store()
def _getBasicTrackFormat(choices, tnChoiceIndex=1, genomeChoiceIndex=0): genome = GeneralGuiTool._getGenomeChoice(choices, genomeChoiceIndex)[0] tn = GeneralGuiTool._getTrackChoice(choices, tnChoiceIndex)[0] from quick.application.GalaxyInterface import GalaxyInterface from gold.description.TrackInfo import TrackInfo from quick.application.ExternalTrackManager import ExternalTrackManager from gold.track.TrackFormat import TrackFormat if ExternalTrackManager.isGalaxyTrack(tn): geSource = ExternalTrackManager.getGESourceFromGalaxyOrVirtualTN( tn, genome) try: tf = GeneralGuiTool._convertToBasicTrackFormat( TrackFormat.createInstanceFromGeSource( geSource).getFormatName()) except Warning: return genome, tn, '' else: if GalaxyInterface.isNmerTrackName(genome, tn): tfName = 'Points' else: tfName = TrackInfo(genome, tn).trackFormatName tf = GeneralGuiTool._convertToBasicTrackFormat(tfName) return genome, tn, tf
def validateRegAndBinSpec(self, regSpec, binSpec): from quick.util.CommonFunctions import convertTNstrToTNListFormat from gold.description.TrackInfo import TrackInfo from gold.track.TrackFormat import TrackFormatReq trackName = convertTNstrToTNListFormat(binSpec) ti = TrackInfo(self._genome, trackName) if not ti.isValid(): return 'The specified track is not valid: "%s"' % ':'.join( trackName) if not (ti.trackFormatName and TrackFormatReq(name=ti.trackFormatName).isInterval()): return 'The specified track does not contain regions: "%s"' % ':'.join( trackName)
def isValidTrack(genome, trackName, fullAccess=False): if not TrackInfo(genome, trackName).isValid(fullAccess): return False if ProcTrackOptions._hasPreprocessedFiles(genome, trackName): return True return False
def constructId(geSource): from gold.origdata.PreProcessTracksJob import PreProcessTracksJob if geSource.hasOrigFile(): origPath = os.path.dirname(geSource.getFileName()) if not geSource.isExternal() else geSource.getFileName() return TrackInfo.constructIdFromPath(geSource.getGenome(), origPath, \ geSource.getVersion(), PreProcessTracksJob.VERSION) else: return geSource.getId()
def constructId(geSource): from gold.origdata.PreProcessTracksJob import PreProcessTracksJob if geSource.hasOrigFile(): origPath = os.path.dirname(geSource.getFileName()) if not geSource.isExternal() else geSource.getFileName() return TrackInfo.constructIdFromPath(geSource.getGenome(), origPath, \ geSource.getVersion(), PreProcessTracksJob.VERSION) else: return None
def getUniqueKey(self, genome): assert( not None in [self._trackFormatReq.allowOverlaps(), \ self._trackFormatReq.borderHandling()] ) if not self._trackId: self._trackId = TrackInfo(genome, self.trackName).id return hash((tuple(self.trackName), self._trackId, self._trackFormatReq.allowOverlaps(), \ self._trackFormatReq.borderHandling()))
def getTrackFormat(cls,genome,trackName) : #trackName here is a list of directories which is path of track #temp = trackName.split(":") temp = trackName if temp[-1] == "-- All subtypes --" : trackName = temp[:-1] else : trackName = temp #trackName = self.trackPrepare(trackName) return TrackInfo(genome,trackName).trackFormatName
def _getValueTypeName(choices, tnChoiceIndex=1, genomeChoiceIndex=0): genome = GeneralGuiTool._getGenomeChoice(choices, genomeChoiceIndex)[0] tn = GeneralGuiTool._getTrackChoice(choices, tnChoiceIndex)[0] from quick.application.GalaxyInterface import GalaxyInterface from gold.description.TrackInfo import TrackInfo from quick.application.ExternalTrackManager import ExternalTrackManager from gold.track.TrackFormat import TrackFormat if ExternalTrackManager.isGalaxyTrack(tn): geSource = ExternalTrackManager.getGESourceFromGalaxyOrVirtualTN( tn, genome) valTypeName = TrackFormat.createInstanceFromGeSource( geSource).getValTypeName() else: if GalaxyInterface.isNmerTrackName(genome, tn): valTypeName = '' else: valTypeName = TrackInfo(genome, tn).markType return valTypeName.lower()
def _compute(self): tv = self._children[0].getResult() if not self._forceIsDirected: from gold.track.RandomizedTrack import RandomizedTrack origTrackName = self._track.trackName[:-2] if isinstance(self._track, RandomizedTrack) else self._track.trackName isDirected = not (TrackInfo(self._region.genome, origTrackName).undirectedEdges == True) else: isDirected = self._isDirected return LazyProtoGraphView.createInstanceFromTrackView(tv, isDirected=isDirected)
def _filterValueListAndMakeTrackInfoObjects(self, category, valueList, genome, trackName): prunedValueList = [] trackNames = [trackName+[v[1]]for v in valueList] for index, trackTuple in enumerate(trackNames): trackInfoObj = TrackInfo(genome, trackTuple) if trackInfoObj.fileType =='': webObject, paramForm = self._getWebPageAndForm( self._makeUrlstreng(self._sessionId, 'hgta_table', trackTuple[-1]) ) self._storeUcscInfoBox(genome, paramForm, trackTuple) if set(['bed', 'wigBed']) & set(['.']+[item.name for item in paramForm.find_control("hgta_outputType").items]): lastForm = self._getForm(mechanize.urlopen(paramForm.click('hgta_doFilterPage'))) if len([control.name for control in lastForm.controls if control.name.find('_.maxOutput')>0]) == 0: prunedValueList.append(valueList[index]) trackInfoObj.fileType = 'bed' trackInfoObj.fileType = 'ucscdb' if trackInfoObj.fileType == '' else 'bed' elif trackInfoObj.fileType =='bed': prunedValueList.append(valueList[index]) trackInfoObj.store() return prunedValueList
def _getBasicTrackFormat(choices, tnChoiceIndex=1): from quick.application.GalaxyInterface import GalaxyInterface from gold.description.TrackInfo import TrackInfo genome = choices[0] tn = choices[tnChoiceIndex].split(':') if GalaxyInterface.isNmerTrackName(genome, tn): tfName = 'Points' else: tfName = TrackInfo(genome, tn).trackFormatName tfName = tfName.lower() if tfName.startswith('linked '): tfName = tfName[7:] tfName = tfName.replace('unmarked ','') tfName = tfName.replace('marked','valued') return genome, tn, tfName
def shouldPreProcessGESource(cls, trackName, geSource, allowOverlaps): genome = geSource.getGenome() storedInfo = TrackInfo(genome, trackName) validFilesExist = cls.preProcFilesExist(genome, trackName, allowOverlaps) and \ storedInfo.isValid() if not geSource.hasOrigFile(): return False if validFilesExist or geSource.isExternal() else True from gold.origdata.PreProcessTracksJob import PreProcessTracksJob if DebugConfig.VERBOSE: print os.linesep.join([ "GenomeElementSource id: {}, ".format( cls.constructId(geSource)) + "TrackInfo.id: {}".format(storedInfo.id), "GenomeElementSource version: {}, ".format( geSource.getVersion()) + "TrackInfo.geSourceVersion: {}".format( storedInfo.geSourceVersion), "PreProcessTracksJob version: {}, ".format( PreProcessTracksJob.VERSION) + "TrackInfo.preProcVersion: {}".format( storedInfo.preProcVersion), ]) storedAsAccordingToGeSource = \ (cls.constructId(geSource) == storedInfo.id and geSource.getVersion() == storedInfo.geSourceVersion and PreProcessTracksJob.VERSION == storedInfo.preProcVersion) #from gold.application.LogSetup import logMessage #logMessage(geSource.getGenome()) #logMessage(':'.join(trackName)) #logMessage('%s %s %s %s %s' % (cls.preProcFilesExist(genome, trackName, allowOverlaps), \ # storedInfo.isValid(), \ # geSource.hasOrigFile(), \ # cls.constructId(geSource) == storedInfo.id, \ # geSource.getVersion() == storedInfo.preProcVersion)) return not (validFilesExist and storedAsAccordingToGeSource)
def _getFileFormatInfo(cls, choices, gSuite, genome, track): if choices.changeFormat == cls.OUTPUT_FORMAT_ORIGINAL: suffix = TrackInfo(genome, track.trackName).fileType fileFormatName = \ getComposerClsFromFileSuffix(suffix).FILE_FORMAT_NAME asOriginal = True allowOverlaps = True return FileFormatInfo(fileFormatName, asOriginal, allowOverlaps, suffix) else: outputFormatDict = cls._getOutputFormatDict(gSuite, genome) return outputFormatDict[choices.outputFormat]
def getUniqueKey(self, genome): if not self._trackId: self._trackId = TrackInfo(genome, self.trackName).id return hash( (tuple(self.trackName), self._trackId if self._trackId else '', getClassName(self.formatConverters[0]) if self.formatConverters else '', self.formatConverters[0].VERSION if self.formatConverters else '', self._trackFormatReq.allowOverlaps() if self._trackFormatReq.allowOverlaps() else '', self._trackFormatReq.borderHandling() if self._trackFormatReq.borderHandling() else ''))
def getSubtypes(genome, trackName, fullAccess=False): dirPath = createDirPath(trackName, genome) subtypes = [fn for fn in ProcTrackOptions._getDirContents(genome, trackName) \ if not (fn[0] in ['.','_'] or os.path.isfile(dirPath + os.sep + fn) \ or GenomeInfo.isValidChr(genome, fn))] if not fullAccess and not ProcTrackOptions._isLiteratureTrack( genome, trackName): subtypes = [ x for x in subtypes if x not in ['external'] and not TrackInfo(genome, trackName + [x]).private ] return sorted(subtypes, key=smartStrLower)
def shouldPreProcessGESource(trackName, geSource, allowOverlaps): storedInfo = TrackInfo(geSource.getGenome(), trackName) validFilesExist = PreProcessUtils.preProcFilesExist(trackName, geSource, allowOverlaps) and \ storedInfo.isValid() if not geSource.hasOrigFile(): return False if validFilesExist or geSource.isExternal() else True storedAsAccordingToGeSource = \ (PreProcessUtils.constructId(geSource) == storedInfo.id and \ geSource.getVersion() == storedInfo.preProcVersion) #from gold.application.LogSetup import logMessage #logMessage(geSource.getGenome()) #logMessage(':'.join(trackName)) #logMessage('%s %s %s %s %s' % (PreProcessUtils.preProcFilesExist(trackName, geSource, allowOverlaps), \ # storedInfo.isValid(), \ # geSource.hasOrigFile(), \ # PreProcessUtils.constructId(geSource) == storedInfo.id, \ # geSource.getVersion() == storedInfo.preProcVersion)) return not (validFilesExist and storedAsAccordingToGeSource)
def _findTrackInfoBasedMetaData(self): if not self._foundTrackInfoBasedMetaData: if PreProcMetaDataCollector.hasKey(self._genome, self._trackName): collector = PreProcMetaDataCollector(self._genome, self._trackName) self._fileSuffix = collector.getFileSuffix() self._geSourceVersion = collector.getGeSourceVersion() self._id = collector.getId() self._undirectedEdges = True if collector.hasUndirectedEdges() else False else: ti = TrackInfo(self._genome, self._trackName) self._fileSuffix = ti.fileType self._geSourceVersion = ti.geSourceVersion self._id = ti.id self._undirectedEdges = True if ti.undirectedEdges else False
def _calcAndStoreSubTrackCount(self, trackName): ti = TrackInfo(self._genome, trackName) trackCount = 0 for subTrackName in ProcTrackOptions.getSubtypes( self._genome, trackName, True): subTrackCount = TrackInfo(self._genome, trackName + [subTrackName]).subTrackCount if subTrackCount: trackCount += subTrackCount if ti.isValid(): trackCount += 1 ti.subTrackCount = trackCount ti.store()
def shouldPreProcessGESource(trackName, geSource, allowOverlaps): genome = geSource.getGenome() storedInfo = TrackInfo(genome, trackName) validFilesExist = PreProcessUtils.preProcFilesExist(genome, trackName, allowOverlaps) and \ storedInfo.isValid() if not geSource.hasOrigFile(): return False if validFilesExist or geSource.isExternal() else True storedAsAccordingToGeSource = \ (PreProcessUtils.constructId(geSource) == storedInfo.id and \ geSource.getVersion() == storedInfo.preProcVersion) #from gold.application.LogSetup import logMessage #logMessage(geSource.getGenome()) #logMessage(':'.join(trackName)) #logMessage('%s %s %s %s %s' % (PreProcessUtils.preProcFilesExist(genome, trackName, allowOverlaps), \ # storedInfo.isValid(), \ # geSource.hasOrigFile(), \ # PreProcessUtils.constructId(geSource) == storedInfo.id, \ # geSource.getVersion() == storedInfo.preProcVersion)) return not (validFilesExist and storedAsAccordingToGeSource)
def removeUnusedRecords(): trackInfoShelve = safeshelve.open(SHELVE_FN, 'w') iremoved = 0 ifound = 0 for key in trackInfoShelve.keys(): try: ti = TrackInfo.createInstanceFromKey(key) fn = ti.getOrigFn() if not os.path.exists(fn): raise Exception('Should exclude nmer tracks and other tracks without standardized track (e.g. intensity tracks). How? Not sure..') ti.removeEntryFromShelve() iremoved = iremoved + 1 else: ifound= ifound + 1 except Exception, e: print "Something wrong with ", fn , ", ", e
def updateOrCreateStandardFolders(genome): print "update or create ", genome for x in standardFolderInfo: ti = TrackInfo(genome, x[0]) ti.description = x[1] ti.reference = x[2] ti.quality = x[3] ### sjekke om folderen eksisterer, hvis ikke lage den. Dette er ikke helt implementer da jeg er usikker om jeg fikk det til med eksisterende metoder. if not getOrigFns( genome, x[0], '' ): # returns empty list if no folder? But what does it return when no tracks is in the folder? print x[0], "is missing" # lage folder og preprocess? #print "ti=", ti ti.store()
def removeUnusedRecords(): trackInfoShelve = safeshelve.open(SHELVE_FN, 'w') iremoved = 0 ifound = 0 for key in trackInfoShelve.keys(): try: ti = TrackInfo.createInstanceFromKey(key) fn = createOrigPath(ti.genome, ti.trackName) if not os.path.exists(fn): raise Exception( 'Should exclude nmer tracks and other tracks without standardized track (e.g. intensity tracks). How? Not sure..' ) ti.removeEntryFromShelve() iremoved = iremoved + 1 else: ifound = ifound + 1 except Exception, e: print "Something wrong with ", fn, ", ", e
def _appendConverterOptions(self, track, labelKey): if track is None: return if track.formatConverters is None: # May happen in the second track object if one analyses a track versus itself return if self.getChoice(labelKey) is not None: assert(self.getChoice(labelKey) == getClassName(track.formatConverters[0])) return labelPair = (labelKey, '_Treat ' + prettyPrintTrackName(track.trackName) + ' as') choicePairs = [ ( getClassName(fc), fc.getOutputDescription(TrackInfo(self._genome, track.trackName).trackFormatName) ) \ for fc in track.formatConverters ] text = '[' + ':'.join(labelPair) + '=' + '/'.join( [':'.join(x) for x in choicePairs] ) + ']' self._analysisParts.append(AnalysisOption( text ))
def _calcAndStoreSubTrackCount(self, trackName): ti = TrackInfo(self._genome, trackName) if ti.isValid(): ti.subTrackCount = 1 ti.store()
def storeTrackInfo(genome, trackName, metaData): trackInfoObject = TrackInfo(genome, trackName) for i in metaData.keys(): setattr(trackInfoObject, i, metaData[i]) trackInfoObject.store()
illegalchars = ''.join(c for c in ''.join(testname) if c not in valid_chars) if len(illegalchars)>0: print 'Illegal characters found in chromosome names "%s". Please rename using legal characters "%s".' % (illegalchars,valid_chars) ''' ''' import string valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) name = "vegard_hei" illegalchars = ''.join(c for c in name if c not in valid_chars) if len(illegalchars)>0: print "Illegal characters found in chromosome names '%s'. Please rename using legal characters " % (illegalchars,valid_chars) ''' ''' import re a = re.search("[ /]", "abc/def") if a: print "treff" ''' from gold.description.TrackInfo import TrackInfo genome="hg18" tn = ['Private', 'Vegard', 'test', 'test1'] ti = TrackInfo(genome, tn) print(ti.getUIRepr()) print "ferdig"
import sys import third_party.safeshelve as safeshelve from gold.description.TrackInfo import TrackInfo from gold.description.TrackInfo import SHELVE_FN from gold.origdata.PreProcessTracksJob import PreProcessAllTracksJob import re #print "SHELVE_FN=", SHELVE_FN trackInfoShelve = safeshelve.open(SHELVE_FN, 'w') allkeys=trackInfoShelve.keys() trackInfoShelve.close() count = 0 for key in allkeys: #print key, count try: ti = TrackInfo.createInstanceFromKey(key) if ti.timeOfPreProcessing > datetime.datetime(2011, 11, 8, 23,0,0) and ti.timeOfPreProcessing < datetime.datetime(2011, 11, 25, 23,0,0): if re.search('Nmers|Trashcan|external|Restriction', ti.getOrigFn())==None: # Nmers, external print 'trying to repair track ', ti.genome, ti.trackName, ti.timeOfPreProcessing count = count +1 ### Sette ID til None og preprocesse. #ti.id = None #ti.store() #PreProcessAllTracksJob(ti.genome, ti.trackName).process() #### printe ut kommandoer for batch job, virket ikke. #print 'key =\''+key+'\'' #print 'ti = TrackInfo.createInstanceFromKey(key)'
def getRunDescription(trackName1, trackName2, trackNameIntensity, analysisDef, ubSource, revEngBatchLine, \ urlForTrackAutoSelection, manualSeed, **kwArgs): genome = ubSource.genome core = HtmlCore() analysis = Analysis(analysisDef, genome, trackName1, trackName2, **kwArgs) core.header('GENOME') core.append(GenomeInfo(genome).mainInfo(printEmpty=False)) core.divider() formatChoices = analysis.getFormatConverterChoicesAsText().items() tr1FormatChoice, tr2FormatChoice = formatChoices if len(formatChoices) == 2 else (None, None) first = True for tn,label,formatChoice in zip([trackName1,trackName2,trackNameIntensity], \ ['TRACK 1','TRACK 2','INTENSITY TRACK'], \ [tr1FormatChoice,tr2FormatChoice,None]): if tn in [None, []]: continue if not first: core.divider() core.header(label) trackInfo = TrackInfo(genome, tn) trackText = '' if ExternalTrackManager.isHistoryTrack(tn): assert len(tn)>=4, 'Length of external track name < 4: %s' % str(tn) core.descriptionLine('Name', ExternalTrackManager.extractNameFromHistoryTN(tn) + ' (from history)' + os.linesep) else: core.descriptionLine('Name', ':'.join(tn) + os.linesep) core.append(trackInfo.mainInfo(printEmpty=False)) if formatChoice is not None: core.descriptionLine('Treated as', formatChoice[1]) first = False core.divider() core.header('ANALYSIS') core.paragraph( ''.join(str(analysis).split(':')[1:]) ) first = True for label,choice in analysis.getInterfaceChoicesAsText().items(): if first: core.divider() core.header('OPTIONS') if manualSeed is not None and label == 'Random seed' and choice == 'Random': choice = str(manualSeed) core.descriptionLine(label, choice) first = False h0 = analysis.getH0() if h0 is not None: core.divider() core.header('NULL HYPOTHESIS') core.paragraph(h0) h1 = analysis.getH1() if h1 is not None: core.divider() core.header('ALTERNATIVE HYPOTHESIS') core.paragraph(h1) core.divider() core.header('ANALYSIS REGIONS') if hasattr(ubSource, 'description'): core.paragraph(ubSource.description) core.divider() core.header('SOLUTION') statClass = analysis.getStat() #One alternative is to put getDescription in MagicStatFactory-hierarchy as class-method, and get real class behind partial-object. #if isinstance(statClass, functools.partial): #statClass = statClass.func #core.paragraph( statClass.getDescription() ) #Chosen alternative is to Instantiate an object, which will automatically give object of real class.. #and then use the following two lines, which will get class in Statistic-hierarchy instead of MagicStatFactory-hierarchy .. try: reg = ubSource.__iter__().next() except: core.paragraph('Solution not relevant, as there are no specified analysis regions..') else: track1, track2 = analysis.getTracks() if statClass is None: core.paragraph('Solution not available, due to currently invalid analysis') logMessage('Solution not available, with params: ' + str([trackName1, trackName2, analysisDef]), level=logging.WARN ) else: statObj = statClass(reg,track1, track2) statDescr = statObj.getDescription() replPat = '<a href=' + os.sep.join([STATIC_REL_PATH,'notes','stats','']) + r'\1>note</a>' statDescr = re.sub('<note>(.*)</note>', replPat, statDescr) core.paragraph( statDescr ) core.divider() core.header('TIME OF ANALYSIS') core.paragraph('Analysis initiated at time: ' + str( datetime.datetime.now() ) ) if urlForTrackAutoSelection not in [None, '']: core.divider() core.header('URL FOR TRACK AUTOSELECTION') #urlOptions = '&'.join(['track1=' + quote(':'.join(trackName1)), 'track2=' + quote(':'.join(trackName2))]) #core.paragraph(URL_PREFIX + '/hyper?' + urlOptions) core.styleInfoBegin(styleClass='break-word') core.paragraph(urlForTrackAutoSelection) core.styleInfoEnd() if revEngBatchLine not in [None, '']: core.divider() core.header('CORRESPONDING BATCH-RUN LINE') #if any(ExternalTrackManager.isRedirectOrExternalTrack(tn) for tn in [trackName1, trackName2]): #core.paragraph('Batch-run line not available with tracks from history') #else: core.styleInfoBegin(styleClass='break-word') core.paragraph(revEngBatchLine) core.styleInfoEnd() core.divider() core.header('REFERENCES') core.paragraph('The HyperBrowser system is described in:<br>"Sandve et al., <a href="http://genomebiology.com/2010/11/12/R121/">The Genomic HyperBrowser: inferential genomics at the sequence level</a>, Genome Biol. 2010;11(12):R121') from gold.statistic.RandomizationManagerStat import RandomizationManagerStat if statClass is not None and RandomizationManagerStat.getMcSamplingScheme(statClass.keywords) == 'MCFDR': core.paragraph('The p-values of this analysis were computed using the MCFDR scheme for Monte Carlo based p-value computation'+\ ', described in:<br>Sandve et al., <a href="http://bioinformatics.oxfordjournals.org/content/early/2011/10/13/bioinformatics.btr568.long">Sequential Monte Carlo multiple testing</a>, Bioinformatics 2011') # description = \ #''' #Run descriptions will be introduced in the next version of HB. <br> #Below is an example run description, which is a static text unconnected to your choices. The purpose is to get feedback from you on what this should look like:<br> #Track1 (refseg:genes): Unmarked points (converted from unmarked segments, taking midpoints)<br> #Track2 (DNA melting:meltmap): Function<br> #Bins: Chr1, divided into bins of 10 megabases<br> #Question: Are track1-points occurring with different frequency inside track2-segment than outside?<br> #Analysis:<br> #The main result is a p-value resulting from a statistical test connected to the question.<br> #The null-hypothesis assumes that the track1-points are randomly distributed according to a poisson-distribution, with the same number of points as in the original data. Track2-segment are assumed fixed as they are in the original data. This can be answered by a binomial test. The alternative hypothesis is then that the count of points inside segments has resulted from a different distribution of points, where the points are then either distributed more or less inside segments versus outside. See the note on this question in the user guide for further info.<br> #''' return str(core)
def handleTask(self, task): task.cls.createTrackChr(task.genome, task.inTrackName, task.outTrackName, task.windowSize, task.func, task.chr) #todo: rewrite to new preprocessor solution... TrackInfo.finalizeTrack(task.genome, task.outTrackName, ['val'], 'float64', 1)
print ' ' + str(e).strip() sys.exit(1) for allowOverlaps in [False, True]: fromDir = createDirPath(['GESourceTracks'], 'TestGenome', allowOverlaps=allowOverlaps) toDir = createDirPath([], 'ModelsForExternalTracks', allowOverlaps=allowOverlaps) try: if not os.path.exists(toDir): shutil.copytree(fromDir, toDir) print 'OK: Copied from %s to %s.' % (fromDir, toDir) except Exception, e: print 'FAILED: Error occurred copying from %s to %s: ' % (fromDir, toDir) + str(e).strip() sys.exit(1) for track in ProcTrackOptions.getSubtypes('TestGenome', ['GESourceTracks']): ti = TrackInfo('TestGenome', ['GESourceTracks', track]) ti.trackName = [track] ti.genome = 'ModelsForExternalTracks' ti.store() from quick.util.GenomeInfo import GenomeInfo from datetime import datetime gi = GenomeInfo('TestGenome') gi.fullName = 'TestGenome' gi.sourceUrls = ['http://hgdownload.cse.ucsc.edu/goldenPath/hg18/chromosomes/chr21.fa.gz', \ 'http://hgdownload.cse.ucsc.edu/goldenPath/hg18/chromosomes/chrM.fa.gz'] gi.sourceChrNames = ['chr21', 'chrM'] gi.installedBy = 'Setup.py' gi.genomeBuildSource = 'NCBI' gi.genomeBuildName = 'hg18' gi.species = 'H**o Sapiens'