def get_prop(self,pkg,prop):
self.loadc();
if(not self.has(pkg)):
raise exceptions.NameError("package has no inv entry" + str((pkg,prop)));
if(not self.contents[pkg].has_key(prop)):
raise exceptions.NameError("package in inv does not have prop" + str((pkg,prop)));
return self.contents[pkg][prop];
def set_prop(self, pkg, prop, val):
self.loadc()
if (not self.has(pkg)):
raise exceptions.NameError("package has no inv entry" +
str((pkg, prop)))
self.contents[pkg][prop] = val
self.savec()
def getEffect(name):
"""
This function finds the appropriate effect in the post-processing chain
"""
ch = chain()
for e in ch:
if e.name == name:
return e
raise exceptions.NameError(name)
return None
def __getattr__(self, attr): ## implicitly wrap methods from plotItem
if hasattr(self.mPlotItem, attr):
m = getattr(self.mPlotItem, attr)
if hasattr(m, '__call__'):
return m
raise exceptions.NameError(attr)
def save_data(self):
if os.path.exists(self.output_dir) == False:
os.mkdir(self.output_dir)
imgH = self.img_height
imgW = self.img_width
imgChannel = self.img_channel
batch_num = self.batch_num
pixelNum=imgH * imgW
imgSize=pixelNum*imgChannel
print "test data pattern: %s" % self.test_pattern
if self.test_img_list:
print 'test image list file: %s' % self.test_img_list
list_file = open(self.test_img_list,'r')
img_files = list_file.readlines()
list_file.close()
img_files =[f[:-1] if f[-1]=='\n' else f for f in img_files]
img_files = [os.path.join(self.test_dir,f) for f in img_files]
else:
img_files=[]
for dirname,subdirnames,filenames in os.walk(self.test_dir):
img_files += [os.path.join(dirname,f) for f in filenames if fnmatch.fnmatch(f, self.test_pattern) ]
# img_files = [f for f in os.listdir(self.test_dir) if os.path.isfile(os.path.join(self.test_dir, f)) and fnmatch.fnmatch(f, self.test_pattern)]
list_file_path = os.path.join(self.test_dir,'img_list.txt')
list_file=open(list_file_path,'w')
# write relative path
for f in img_files:
list_file.write(f[len(self.test_dir)+1:]+'\n')
# list_file.writelines(img_files)
list_file.close()
print 'find %d images' % len(img_files)
print img_files[:10]
# img_files=[f for f in os.listdir(self.test_dir) if os.path.isfile(os.path.join(self.test_dir,f))]
test_meta={}
test_meta['img_files']=img_files
if self.test_label_file:
pickle(os.path.join(self.output_dir, 'val_batches.meta'), test_meta, True)
else:
pickle(os.path.join(self.output_dir,'pred_batches.meta'),test_meta,True)
imgN = len(img_files)
print "find %d images" % imgN
if self.test_label_file:
labelFile = open(self.test_label_file, "r")
imgIdx = 0
for i in range(batch_num):
bstart = time.clock()
if i < (batch_num - 1):
batchSize = imgN / batch_num
else:
batchSize = imgN - (batch_num - 1) * (imgN / batch_num)
print "%d th batch out of %d batches has %d images" % (i, batch_num, batchSize)
# On exit, batchData has shape (imgSize, batchSize)
batchData = np.zeros((imgChannel, imgH, imgW, batchSize), dtype=np.uint8)
batchDataView= np.swapaxes(batchData,0,3)
if self.test_label_file:
batchLabel = np.zeros((batchSize), dtype=np.float32)
for j in range(batchSize):
if j % 1000 == 0:
print "progress %3.2f" % (float(100 * j) / float(batchSize))
imgFile = img_files[imgIdx]
imgFile = os.path.join(self.test_dir, imgFile)
try:
im = misc.imread(imgFile)
except Exception, e:
print "fail to read image: %s" % imgFile
print e
sys.exit()
# typically, im.shape=(imgH,imgW,imgChannel)
if im.shape[0] != imgH or im.shape[1] != imgW:
print 'im shape'
print im.shape
print 'expected image shape: (height,width)=(%d,%d)' % (imgH,imgW)
raise ecp.NameError('incorrect image shape')
if im.shape[2]==3:
if imgChannel != 3:
raise ecp.NameError('incorrect image color channels')
batchDataView[j][:][:][:]=im
elif im.shape[2] == 1:
print "a grey image"
for k in range(imgChannel):
batchDataView[j][:][:][k]=im
else:
raise ecp.NameError('non-RGB non-L images are not handled')
if self.test_label_file:
batchLabel[j] = float(labelFile.readline()) - 1 # label ranges from 0 to num_classes-1
imgIdx += 1
batchData=np.reshape(batchData, (imgSize,batchSize))
batch_dic = {}
batch_dic['data'] = batchData
if self.test_label_file:
batch_dic['labels'] = batchLabel
print "pickle batch data"
batchFilePath = os.path.join(self.output_dir, self.batch_fname + '_' +
str(self.batch_start_index + i))
print "batchFilePath: %s" % batchFilePath
pickle(batchFilePath, batch_dic,True)
bend = time.clock()
print "batch elapsed time %f" % (bend - bstart)
def save_data(self):
if os.path.exists(self.output_dir) == False:
os.mkdir(self.output_dir)
# generate data files and meta data file
batch_num = self.batch_num
batch_fname = self.batch_fname
img_height = self.img_height
img_width = self.img_width
img_channel = self.img_channel
pixelNum = img_height * img_width
imgSize = pixelNum * img_channel
matlab_meta = scipy.io.loadmat(self.input_matlab_meta_data_file)
synsets = matlab_meta['synsets']
# costMat = matlab_meta['cost_matrix']
synsetsN = len(synsets)
print "synsets length:%d" % len(synsets)
wnID_2_ILSVRSID = {};
label_2_labelname = [None] * len(synsets)
for i in range(len(synsets)):
wnID_2_ILSVRSID[str(synsets[i][0][1][0])] = synsets[i][0][0][0][0]
label_2_labelname[i] = str(synsets[i][0][2][0])
# print "wnID_2_ILSVRSID"
# print wnID_2_ILSVRSID
wnID = [f for f in os.listdir(self.train_dir) if os.path.isdir(os.path.join(self.train_dir, f))]
wnNum = len(wnID)
classSz = np.zeros((wnNum), dtype=np.uint32) # classes at leaf nodes of semantic tree
wnID2label = {}
for i in range(wnNum):
label = -1;
for j in range(synsetsN):
if str(synsets[j][0][1][0]) == wnID[i]:
label = j # label starts from 0
break
if label < 0:
print "wnID %s is not found in synsets " % wnID[i]
raise ecp.NameError('error')
wnID2label[wnID[i]] = label
classSz[i] = synsets[label][0][7][0][0]
totalImgN = sum(classSz)
print 'totally %d images' % totalImgN
# need to sample a subset of images because main memory can not hold all pixels in training images
PCA_img_num = self.PCA_img_num
if PCA_img_num > totalImgN:
PCA_img_num = totalImgN
print "use %d images to compute pixelwise PCA" % PCA_img_num
imgID2Name = [None] * totalImgN
imgID2Label = np.zeros((totalImgN), dtype=np.float32)
classMember = [None] * wnNum
num_image_correct = 1
imgIdx = 0
maxLabel = -1
for i in range(wnNum):
label = wnID2label[wnID[i]];
if label > maxLabel:
maxLabel = label
subDir = wnID[i]
subDirPath = os.path.join(self.train_dir, subDir)
imgs = [img for img in os.listdir(subDirPath) if os.path.isfile(os.path.join(subDirPath, img)) and fnmatch.fnmatch(img, self.train_pattern)]
if classSz[i] != len(imgs):
print "wnID:%s %d images in folder and %d images from synset" % (wnID[i], len(imgs), classSz[i])
num_image_correct = 0
lClsMember = [];
for j in range(len(imgs)):
imgID2Name[imgIdx] = os.path.join(subDirPath, imgs[j])
imgID2Label[imgIdx] = label
lClsMember += [imgIdx]
imgIdx = imgIdx + 1
classMember[i] = lClsMember
print 'max label : %d' % maxLabel
if num_image_correct == 0:
raise ecp.NameError('incorrect training images number')
batch_size = np.zeros((wnNum, batch_num), dtype=np.uint32)
for i in range(wnNum):
batch_size[i][:] = get_batch_size(classSz[i], batch_num)
# randomly distribute images to batches in a class-wise manner
for i in range(len(classMember)):
rd.shuffle(classMember[i])
# dataMean has shape (img_channel,img_height,img_width)
dataMean = np.zeros((img_channel, img_height, img_width), dtype=np.float64)
# dataMeanView has shape (img_height, img_width, img_channel)
dataMeanView = dataMean.swapaxes(0, 2).swapaxes(0, 1)
for i in range(batch_num):
batchFileFn = batch_fname + '_' + str(i + self.batch_start_index)
batchFileFn = os.path.join(self.output_dir, batchFileFn)
print "%d th batch file name:%s" % (i, batchFileFn)
batchImgN = np.sum(batch_size[:, i])
print "%d images in %d th batch out of %d batches" % (batchImgN, i, batch_num)
for i in range(batch_num):
batchFileFn = batch_fname + '_' + str(i + self.batch_start_index)
batchFileFn = os.path.join(self.output_dir, batchFileFn)
print "%d th batch file name:%s" % (i, batchFileFn)
bstart = time.clock()
batchImgN = np.sum(batch_size[:, i])
print "%d images in %d th batch out of %d batches" % (batchImgN, i, batch_num)
# On exit, batchData has shape (imgSize, batchImgN)
batchData = np.zeros((img_channel, img_height, img_width, batchImgN), dtype=np.uint8)
# batchDataView shape (batchImgN, img_height, img_width, img_channel)
batchDataView = np.swapaxes(batchData, 0, 3)
batchLabel = np.zeros(batchImgN, dtype=np.float32)
batchImgId = []
# batchImgIdx = 0
# collect image ids in the batch
for j in range(wnNum):
batchClsN = batch_size[j, i]
startIdx = np.sum(batch_size[j, 0:i])
batchImgId += classMember[j][startIdx:startIdx + batchClsN]
assert len(batchImgId) == batchImgN
# important! shuffle images in the batch so that mini-batch has random class distribution
rd.shuffle(batchImgId)
for j in range(batchImgN):
lImgID = batchImgId[j]
try:
im = misc.imread(imgID2Name[lImgID])
except Exception, e:
print e
sys.exit()
if im.shape[1] != img_width or im.shape[0] != img_height:
raise ecp.NameError('incorrect image size')
if im.shape[2] == 3:
if img_channel != 3:
raise ecp.NameError('incorrect image color channels')
batchDataView[j][:][:][:] = im
dataMeanView += im
elif im.mode == "L":
print "a grey image"
for p in range(img_channel):
batchDataView[j][:][:][p] = im
dataMeanView[:][:][p] += im
else:
raise ecp.NameError('non-RGB non-gray images are not handled')
batchLabel[j] = imgID2Label[lImgID]
# for j in range(wnNum):
# if j % 200 == 0:
# print "progress : %5.4f " % (float(100 * j) / float(wnNum))
# # if i < batch_num - 1:
# # batchClsN = batchSize[j]
# # else:
# # batchClsN = lastBatchSize[j]
# batchClsN = batch_size[j, i]
#
# # print "batchClsN %d" % batchClsN
# startIdx = np.sum(batch_size[j, 0:i])
# for k in range(batchClsN):
# # lImgID = classMember[j][i * batchSize[j] + k]
# lImgID = classMember[j][startIdx + k]
# try:
# im = misc.imread(imgID2Name[lImgID])
# except Exception, e:
# print e
# sys.exit()
# # typically, im.shape=(imgH,imgW,imgChannel)
# if im.shape[1] != img_width or im.shape[0] != img_height:
# raise ecp.NameError('incorrect image size')
# if im.shape[2] == 3:
# if img_channel != 3:
# raise ecp.NameError('incorrect image color channels')
# batchDataView[batchImgIdx][:][:][:] = im
# dataMeanView += im
# elif im.mode == "L":
# print "a grey image"
# for p in range(img_channel):
# batchDataView[batchImgIdx][:][:][p] = im
# dataMeanView[:][:][p] += im
# else:
# raise ecp.NameError('non-RGB non-gray images are not handled')
#
# batchLabel[batchImgIdx] = imgID2Label[lImgID]
# batchImgIdx += 1
batchData = np.reshape(batchData, (imgSize, batchImgN))
batch_dic = {}
batch_dic['data'] = batchData
batch_dic['labels'] = batchLabel
try:
pickle(batchFileFn, batch_dic,True);
except Exception as inst:
print "exception occurs"
print type(inst)
print inst
bend = time.clock()
print 'batch elapsed time:%f' % (bend - bstart)
def __init__(self, dirPath, obsNames=None,
create=False, refDirPath=None, name=None, ppExePath=None,
ppOutputFile='observations.nc', runTime=None, runCode=None,# options for creating new study
update=False, # options for updating existing study
verbose=False, **parameters):
"""
Create an instance of HadCM3 class. Default behaviour is to read from dirPath and prohibit updates.
:param dirPath -- path to directory where model simulation exists or is to be created
:param create (optional with default False). If True create new directory and populate it.
Afterwards the ModelSimulation will be readOnly.
These options should be specified when creating a new study otherwise they are optional and ignored
:param refDirPath -- reference directory. Copy all files from here into dirPath
:param name -- name of the model simulation. If not provided will be taken from dirPath
:param ppExePath -- path to post proessing executable
:param ppOutputFile -- File name of output of post processing executable. Default is observations.nc
:param obsNames -- list of observations to be readin. (see readObs())
:param runTime -- run time in seconds for UM job. If set to None nothing is changed.
:param runCode -- code to be used by Job.
:param update -- allow updates to the simulation information.
:param verbose -- provide verbose output. (See individual methods). Default is False.
: kwargs -- these are parameter and values.
:returns initialised object.
"""
# no parameters should be provided unless create or update provided
if len(parameters) >0 and not (create or update):
raise exceptions.ValueError("Provided parameters but not specified create or update")
# do HadCM3 specific cross checks.
# should not specifiy ASTART & AINITIAL
if 'AINITIAL' in parameters and 'ASTART' in parameters:
raise exceptions.NameError("DO not specify both AINITIAL and ASTART.")
# and OINITIAL and OSTART
if 'OINITIAL' in parameters and 'OSTART' in parameters:
raise exceptions.NameError("DO not specify both OINITIAL and OSTART.")
# call superclass init
super(HadCM3, self).__init__(dirPath,
obsNames=obsNames, create=create, refDirPath=refDirPath, name=name, ppExePath=ppExePath,
ppOutputFile=ppOutputFile, parameters=parameters, # options for creating new study
update=update, # options for updating existing study
verbose=verbose)
if create: # want to create model instance so do creation.
self.fixClimFCG() # fix the ClimFGC
self.modifySubmit(runTime=runTime, runCode=runCode) # modify the Submit script
self.modifyScript() # modify the script
self.createWorkDir(refDirPath) # create the work dirctory (and fill it in)
## Set up namelist mappings
# easy case all variables in SLBC21 and which just set the values.
for var in ['VF1','ICE_SIZE', 'ENTCOEF','CT', 'ASYM_LAMBDA', 'CHARNOCK','G0', 'Z0FSEA']:
self.simpleNamelist(var)
# Hard case ones where we have a function to run or perturb multiple variables are more complex.
# for meta-parameter we register function. Functions should return dict indexed by namelist info.
# I think these should bs class methods to save computing them every time we create a model.
# but overhead is likely small
# TODO: Convert MetaFn and Namelists to class methods.
self.registerMetaFn('RUNID', runName, verbose=verbose) # runid
self.registerMetaFn('KAY_GWAVE', gravityWave, verbose=verbose) # gravity wave
self.registerMetaFn('ALPHAM',iceAlbedo, verbose=verbose) # ice albedo
self.registerMetaFn('CW_LAND',cloudWater,verbose=verbose) # CW_LAND
self.registerMetaFn('RHCRIT',cloudRHcrit,verbose=verbose) # RHCRIT meta-param generates array
self.registerMetaFn('EACF', cloudEACF, verbose=verbose) # EACF meta-param generates array
self.registerMetaFn('DYNDIFF', diffusion, verbose=verbose) # Dynamics Diffusion generates lots of arrays
self.registerMetaFn('RUN_TARGET',runTarget,verbose=verbose) # length of simulation -- modifies several namelist vars
self.registerMetaFn('START_TIME', startTime, verbose=verbose) # start_Time for run -- modifies several namelist vars
self.registerMetaFn("SPHERICAL_ICE",sph_ice,verbose=verbose) # Spherical ice (or not)
# add AINITIAL, OINITIAL, ASTART & OSTART in inithist
for var in ['AINITIAL', 'OINITIAL', 'ASTART', 'OSTART']: # need slightly special code for these variables.
# ideas tis to use the same basic function which has as argument the parameter. Then use functools.partial to
# generate function which gets registered.
fn=functools.partial(initHist_nlcfiles, parameter=var)
fn.func_name=var.lower()+'_initHist_nlcfiles'
self.registerMetaFn(var, fn, verbose=verbose)
# got some parameters and either creating or updating -- update namelist.
if len(parameters) >0 and (create or update):
self.setReadOnly(False) # allow modification
self.setParams(parameters,verbose=verbose,fail=True) # apply namelist etc
# deal with START_TIME -- f90nml.patch fails... nad f90nml.read doesn't work as it should
# need a temp files etc -- TODO: If need this functionality again then warp in a subroutine and try and fix f90nml!
#
if 'START_TIME' in parameters:
RECONA_FILE=os.path.join(self.dirPath,'RECONA')
recona=f90nml.read(RECONA_FILE)
if f90nml.__version__ != '0.21': # version should be 0.21
raise Exception("Can only work with f90nml version 0.21. Version is %s"%(f90nml.__version__))
# f90nml is .20 then it appears to truncate the array
# so fill it in, # general problem here is that array length varies in rather arbitrary ways.
# the only obvious reference point we have is 405 at fixhs[11] -- which means vn4.5 TODO fix this.
fixhd=recona['headers']['FIXHD']
# f90nml (even at vn 0.21) has problems reading in nml in the following format
# fixhd(12)=xxx -- instead putting the value at posn 0.
offset=fixhd.index(405)-11 # if 405 not found will trigger an error.
print "patching fixhd in %s is "%RECONA_FILE,repr(fixhd)
#all this is done because f90nml seems to remember it starts at some posn.
fixhd[20+offset]=parameters['START_TIME'][0]
fixhd[27+offset]=parameters['START_TIME'][0]
recona.write(RECONA_FILE, force=True) # overwrite the existing file
self.setReadOnly(True) # make it read only