def verify_cluster_results_filename(resultsFilename): """ Checks whether a results file already exists on the cluster, and returns an available version of the results filename. Should be called before writing a new results file. Parameters ---------- resultsFilename : str cluster path, e.g. pyme-cluster:///example_folder/name.h5r Returns ------- resultsFilename : str cluster path which may have _# appended to it if the input resultsFileName is already in use, e.g. pyme-cluster:///example_folder/name_1.h5r """ from PYME.IO import clusterIO import os if clusterIO.exists(resultsFilename): di, fn = os.path.split(resultsFilename) i = 1 stub = os.path.splitext(fn)[0] while clusterIO.exists(os.path.join(di, stub + '_%d.h5r' % i)): i += 1 resultsFilename = os.path.join(di, stub + '_%d.h5r' % i) return resultsFilename
def _getFilenameSuggestion(dirname='', seriesname=SERIES_PATTERN): from PYME.IO.FileUtils import nameUtils from PYME.IO import clusterIO import os if dirname == '': dirname = nameUtils.genClusterDataFilepath() else: dirname = dirname.split(nameUtils.getUsername())[-1] dir_parts = dirname.split(os.path.sep) if len(dirname) < 1 or len(dir_parts) > 3: #path is either too complex, or too easy - revert to default dirname = nameUtils.genClusterDataFilepath() else: dirname = nameUtils.getUsername() + '/'.join(dir_parts) #dirname = defDir % nameUtils.dateDict seriesStub = dirname + '/' + seriesname % nameUtils.dateDict seriesCounter = 0 seriesName = seriesStub % {'counter': nameUtils.numToAlpha(seriesCounter)} #try to find the next available serie name while clusterIO.exists(seriesName + '/'): seriesCounter += 1 if '%(counter)' in seriesName: seriesName = seriesStub % { 'counter': nameUtils.numToAlpha(seriesCounter) } else: seriesName = seriesStub + '_' + nameUtils.numToAlpha(seriesCounter) return seriesName
def mkdir(request, basedir): from PYME.IO import clusterIO newDirectory = request.POST.get('newDirectory', request.GET.get('newDirectory', None)) if newDirectory is None or newDirectory == '': return HttpResponseForbidden('No directory name specified') newDirectory = (basedir + newDirectory).rstrip('/') + '/' if clusterIO.exists(newDirectory) or clusterIO.exists(newDirectory[:-1]): return HttpResponseForbidden('Directory already exists') clusterIO.put_file(newDirectory, '') return HttpResponse(newDirectory)
def check_mapexists(mdh, type='dark'): import os from PYME.IO import clusterIO from PYME.IO.FileUtils import nameUtils from PYME.Analysis.gen_sCMOS_maps import map_filename if type == 'dark': id = 'Camera.DarkMapID' elif type == 'variance': id = 'Camera.VarianceMapID' elif type == 'flatfield': id = 'Camera.FlatfieldMapID' else: raise RuntimeError('unknown map type %s' % type) mapfn = map_filename(mdh, type) # find and record calibration paths local_path = os.path.join( nameUtils.getCalibrationDir(mdh['Camera.SerialNumber']), mapfn) cluster_path = 'CALIBRATION/%s/%s' % (mdh['Camera.SerialNumber'], mapfn) if clusterIO.exists(cluster_path): c_path = 'PYME-CLUSTER://%s/%s' % (clusterIO.local_serverfilter, cluster_path) mdh[id] = c_path return c_path elif os.path.exists(local_path): mdh[id] = local_path return local_path else: return None
def is_complete(self): if not self._complete: # if cached property is false, check to see if anything has changed #TODO - add check to see if we have an updated number of frames self._complete = clusterIO.exists(self.eventFileName, self.clusterfilter) return self._complete
def _verifyClusterResultsFilename(resultsFilename): from PYME.IO import clusterIO if clusterIO.exists(resultsFilename): di, fn = os.path.split(resultsFilename) i = 1 stub = os.path.splitext(fn)[0] while clusterIO.exists(os.path.join(di, stub + '_%d.h5r' % i)): i += 1 fdialog = wx.TextEntryDialog( None, 'Analysis file already exists, please select a new filename') #, fdialog.SetValue(stub + '_%d.h5r' % i) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = os.path.join(di, fdialog.GetValue().encode()) else: raise RuntimeError('Invalid results file - not running') return resultsFilename
def _verifyClusterResultsFilename(resultsFilename): from PYME.IO import clusterIO, unifiedIO resultsFilename = unifiedIO.verbose_fix_name( resultsFilename) # fix any spaces in the input filename if clusterIO.exists(resultsFilename): di, fn = posixpath.split(resultsFilename) i = 1 stub = posixpath.splitext(fn)[0] while clusterIO.exists(posixpath.join(di, stub + '_%d.h5r' % i)): i += 1 fdialog = wx.TextEntryDialog( None, 'Analysis file already exists, please select a new filename') #, fdialog.SetValue(stub + '_%d.h5r' % i) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = posixpath.join(di, str(fdialog.GetValue())) else: raise RuntimeError('Invalid results file - not running') return resultsFilename
def __init__(self, fsdir, virdir, parent=None): if not clusterIO.exists(fsdir): raise RuntimeError("Local directory (fsdir) not found: " + fsdir) self.fsname = fsdir self.name = virdir if self.fsname[-1] != os.sep: if self.fsname[-1] == '/': # fixup win/dos/mac separators self.fsname = self.fsname[:-1] + os.sep else: self.fsname += os.sep self.virname = virdir if self.virname[-1] != '/': self.virname += '/' self.parent = parent self.type = Member.M_COLLECTION
def upload_files(request, directory): from PYME.IO import clusterIO files = request.FILES.getlist('file') for file in files: targetFilename = directory + file.name if clusterIO.exists(targetFilename): return HttpResponseForbidden( 'Upload failed [no files uploaded]. %s already exists on cluster' % targetFilename) for file in files: targetFilename = directory + file.name clusterIO.put_file(targetFilename, file.read()) return HttpResponseRedirect(request.META['HTTP_REFERER'])
def __init__(self, url, queue=None): self.seriesName = url #print url self.clusterfilter = url.split('://')[1].split('/')[0] #print self.clusterfilter self.sequenceName = url.split('://%s/' % self.clusterfilter)[1] #print self.sequenceName self.lastShapeTime = 0 mdfn = '/'.join([self.sequenceName, 'metadata.json']) #print mdfn self.mdh = MetaDataHandler.NestedClassMDHandler() self.mdh.update( json.loads(clusterIO.get_file(mdfn, self.clusterfilter))) self.fshape = None #(self.mdh['Camera.ROIWidth'],self.mdh['Camera.ROIHeight']) self._getNumFrames() # if the series is complete when we start, we don't need to update the number of slices self._complete = clusterIO.exists(self.eventFileName, self.clusterfilter)
def exists(seriesName): return clusterIO.exists(getReducedFilename(seriesName))
def isComplete(self): #TODO - add check to see if we have an updated number of frames return clusterIO.exists(self.eventFileName, self.clusterfilter)