def verify_cluster_results_filename(resultsFilename):
"""
Checks whether a results file already exists on the cluster, and returns an available version of the results
filename. Should be called before writing a new results file.
Parameters
----------
resultsFilename : str
cluster path, e.g. pyme-cluster:///example_folder/name.h5r
Returns
-------
resultsFilename : str
cluster path which may have _# appended to it if the input resultsFileName is already in use, e.g.
pyme-cluster:///example_folder/name_1.h5r
"""
from PYME.IO import clusterIO
import os
if clusterIO.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
i = 1
stub = os.path.splitext(fn)[0]
while clusterIO.exists(os.path.join(di, stub + '_%d.h5r' % i)):
i += 1
resultsFilename = os.path.join(di, stub + '_%d.h5r' % i)
return resultsFilename
def _getFilenameSuggestion(dirname='', seriesname=SERIES_PATTERN):
from PYME.IO.FileUtils import nameUtils
from PYME.IO import clusterIO
import os
if dirname == '':
dirname = nameUtils.genClusterDataFilepath()
else:
dirname = dirname.split(nameUtils.getUsername())[-1]
dir_parts = dirname.split(os.path.sep)
if len(dirname) < 1 or len(dir_parts) > 3:
#path is either too complex, or too easy - revert to default
dirname = nameUtils.genClusterDataFilepath()
else:
dirname = nameUtils.getUsername() + '/'.join(dir_parts)
#dirname = defDir % nameUtils.dateDict
seriesStub = dirname + '/' + seriesname % nameUtils.dateDict
seriesCounter = 0
seriesName = seriesStub % {'counter': nameUtils.numToAlpha(seriesCounter)}
#try to find the next available serie name
while clusterIO.exists(seriesName + '/'):
seriesCounter += 1
if '%(counter)' in seriesName:
seriesName = seriesStub % {
'counter': nameUtils.numToAlpha(seriesCounter)
}
else:
seriesName = seriesStub + '_' + nameUtils.numToAlpha(seriesCounter)
return seriesName
def mkdir(request, basedir):
from PYME.IO import clusterIO
newDirectory = request.POST.get('newDirectory',
request.GET.get('newDirectory', None))
if newDirectory is None or newDirectory == '':
return HttpResponseForbidden('No directory name specified')
newDirectory = (basedir + newDirectory).rstrip('/') + '/'
if clusterIO.exists(newDirectory) or clusterIO.exists(newDirectory[:-1]):
return HttpResponseForbidden('Directory already exists')
clusterIO.put_file(newDirectory, '')
return HttpResponse(newDirectory)
def check_mapexists(mdh, type='dark'):
import os
from PYME.IO import clusterIO
from PYME.IO.FileUtils import nameUtils
from PYME.Analysis.gen_sCMOS_maps import map_filename
if type == 'dark':
id = 'Camera.DarkMapID'
elif type == 'variance':
id = 'Camera.VarianceMapID'
elif type == 'flatfield':
id = 'Camera.FlatfieldMapID'
else:
raise RuntimeError('unknown map type %s' % type)
mapfn = map_filename(mdh, type)
# find and record calibration paths
local_path = os.path.join(
nameUtils.getCalibrationDir(mdh['Camera.SerialNumber']), mapfn)
cluster_path = 'CALIBRATION/%s/%s' % (mdh['Camera.SerialNumber'], mapfn)
if clusterIO.exists(cluster_path):
c_path = 'PYME-CLUSTER://%s/%s' % (clusterIO.local_serverfilter,
cluster_path)
mdh[id] = c_path
return c_path
elif os.path.exists(local_path):
mdh[id] = local_path
return local_path
else:
return None
def is_complete(self):
if not self._complete:
# if cached property is false, check to see if anything has changed
#TODO - add check to see if we have an updated number of frames
self._complete = clusterIO.exists(self.eventFileName,
self.clusterfilter)
return self._complete
def _verifyClusterResultsFilename(resultsFilename):
from PYME.IO import clusterIO
if clusterIO.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
i = 1
stub = os.path.splitext(fn)[0]
while clusterIO.exists(os.path.join(di, stub + '_%d.h5r' % i)):
i += 1
fdialog = wx.TextEntryDialog(
None,
'Analysis file already exists, please select a new filename') #,
fdialog.SetValue(stub + '_%d.h5r' % i)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = os.path.join(di, fdialog.GetValue().encode())
else:
raise RuntimeError('Invalid results file - not running')
return resultsFilename
def _verifyClusterResultsFilename(resultsFilename):
from PYME.IO import clusterIO, unifiedIO
resultsFilename = unifiedIO.verbose_fix_name(
resultsFilename) # fix any spaces in the input filename
if clusterIO.exists(resultsFilename):
di, fn = posixpath.split(resultsFilename)
i = 1
stub = posixpath.splitext(fn)[0]
while clusterIO.exists(posixpath.join(di, stub + '_%d.h5r' % i)):
i += 1
fdialog = wx.TextEntryDialog(
None,
'Analysis file already exists, please select a new filename') #,
fdialog.SetValue(stub + '_%d.h5r' % i)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = posixpath.join(di, str(fdialog.GetValue()))
else:
raise RuntimeError('Invalid results file - not running')
return resultsFilename
def __init__(self, fsdir, virdir, parent=None):
if not clusterIO.exists(fsdir):
raise RuntimeError("Local directory (fsdir) not found: " + fsdir)
self.fsname = fsdir
self.name = virdir
if self.fsname[-1] != os.sep:
if self.fsname[-1] == '/': # fixup win/dos/mac separators
self.fsname = self.fsname[:-1] + os.sep
else:
self.fsname += os.sep
self.virname = virdir
if self.virname[-1] != '/':
self.virname += '/'
self.parent = parent
self.type = Member.M_COLLECTION
def upload_files(request, directory):
from PYME.IO import clusterIO
files = request.FILES.getlist('file')
for file in files:
targetFilename = directory + file.name
if clusterIO.exists(targetFilename):
return HttpResponseForbidden(
'Upload failed [no files uploaded]. %s already exists on cluster'
% targetFilename)
for file in files:
targetFilename = directory + file.name
clusterIO.put_file(targetFilename, file.read())
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def __init__(self, url, queue=None):
self.seriesName = url
#print url
self.clusterfilter = url.split('://')[1].split('/')[0]
#print self.clusterfilter
self.sequenceName = url.split('://%s/' % self.clusterfilter)[1]
#print self.sequenceName
self.lastShapeTime = 0
mdfn = '/'.join([self.sequenceName, 'metadata.json'])
#print mdfn
self.mdh = MetaDataHandler.NestedClassMDHandler()
self.mdh.update(
json.loads(clusterIO.get_file(mdfn, self.clusterfilter)))
self.fshape = None #(self.mdh['Camera.ROIWidth'],self.mdh['Camera.ROIHeight'])
self._getNumFrames()
# if the series is complete when we start, we don't need to update the number of slices
self._complete = clusterIO.exists(self.eventFileName,
self.clusterfilter)
def exists(seriesName):
return clusterIO.exists(getReducedFilename(seriesName))
def isComplete(self):
#TODO - add check to see if we have an updated number of frames
return clusterIO.exists(self.eventFileName, self.clusterfilter)
