def get_prop(self,pkg,prop): self.loadc(); if(not self.has(pkg)): raise exceptions.NameError("package has no inv entry" + str((pkg,prop))); if(not self.contents[pkg].has_key(prop)): raise exceptions.NameError("package in inv does not have prop" + str((pkg,prop))); return self.contents[pkg][prop];
def set_prop(self, pkg, prop, val): self.loadc() if (not self.has(pkg)): raise exceptions.NameError("package has no inv entry" + str((pkg, prop))) self.contents[pkg][prop] = val self.savec()
def getEffect(name): """ This function finds the appropriate effect in the post-processing chain """ ch = chain() for e in ch: if e.name == name: return e raise exceptions.NameError(name) return None
def __getattr__(self, attr): ## implicitly wrap methods from plotItem if hasattr(self.mPlotItem, attr): m = getattr(self.mPlotItem, attr) if hasattr(m, '__call__'): return m raise exceptions.NameError(attr)
def save_data(self): if os.path.exists(self.output_dir) == False: os.mkdir(self.output_dir) imgH = self.img_height imgW = self.img_width imgChannel = self.img_channel batch_num = self.batch_num pixelNum=imgH * imgW imgSize=pixelNum*imgChannel print "test data pattern: %s" % self.test_pattern if self.test_img_list: print 'test image list file: %s' % self.test_img_list list_file = open(self.test_img_list,'r') img_files = list_file.readlines() list_file.close() img_files =[f[:-1] if f[-1]=='\n' else f for f in img_files] img_files = [os.path.join(self.test_dir,f) for f in img_files] else: img_files=[] for dirname,subdirnames,filenames in os.walk(self.test_dir): img_files += [os.path.join(dirname,f) for f in filenames if fnmatch.fnmatch(f, self.test_pattern) ] # img_files = [f for f in os.listdir(self.test_dir) if os.path.isfile(os.path.join(self.test_dir, f)) and fnmatch.fnmatch(f, self.test_pattern)] list_file_path = os.path.join(self.test_dir,'img_list.txt') list_file=open(list_file_path,'w') # write relative path for f in img_files: list_file.write(f[len(self.test_dir)+1:]+'\n') # list_file.writelines(img_files) list_file.close() print 'find %d images' % len(img_files) print img_files[:10] # img_files=[f for f in os.listdir(self.test_dir) if os.path.isfile(os.path.join(self.test_dir,f))] test_meta={} test_meta['img_files']=img_files if self.test_label_file: pickle(os.path.join(self.output_dir, 'val_batches.meta'), test_meta, True) else: pickle(os.path.join(self.output_dir,'pred_batches.meta'),test_meta,True) imgN = len(img_files) print "find %d images" % imgN if self.test_label_file: labelFile = open(self.test_label_file, "r") imgIdx = 0 for i in range(batch_num): bstart = time.clock() if i < (batch_num - 1): batchSize = imgN / batch_num else: batchSize = imgN - (batch_num - 1) * (imgN / batch_num) print "%d th batch out of %d batches has %d images" % (i, batch_num, batchSize) # On exit, batchData has shape (imgSize, batchSize) batchData = np.zeros((imgChannel, imgH, imgW, batchSize), dtype=np.uint8) batchDataView= np.swapaxes(batchData,0,3) if self.test_label_file: batchLabel = np.zeros((batchSize), dtype=np.float32) for j in range(batchSize): if j % 1000 == 0: print "progress %3.2f" % (float(100 * j) / float(batchSize)) imgFile = img_files[imgIdx] imgFile = os.path.join(self.test_dir, imgFile) try: im = misc.imread(imgFile) except Exception, e: print "fail to read image: %s" % imgFile print e sys.exit() # typically, im.shape=(imgH,imgW,imgChannel) if im.shape[0] != imgH or im.shape[1] != imgW: print 'im shape' print im.shape print 'expected image shape: (height,width)=(%d,%d)' % (imgH,imgW) raise ecp.NameError('incorrect image shape') if im.shape[2]==3: if imgChannel != 3: raise ecp.NameError('incorrect image color channels') batchDataView[j][:][:][:]=im elif im.shape[2] == 1: print "a grey image" for k in range(imgChannel): batchDataView[j][:][:][k]=im else: raise ecp.NameError('non-RGB non-L images are not handled') if self.test_label_file: batchLabel[j] = float(labelFile.readline()) - 1 # label ranges from 0 to num_classes-1 imgIdx += 1 batchData=np.reshape(batchData, (imgSize,batchSize)) batch_dic = {} batch_dic['data'] = batchData if self.test_label_file: batch_dic['labels'] = batchLabel print "pickle batch data" batchFilePath = os.path.join(self.output_dir, self.batch_fname + '_' + str(self.batch_start_index + i)) print "batchFilePath: %s" % batchFilePath pickle(batchFilePath, batch_dic,True) bend = time.clock() print "batch elapsed time %f" % (bend - bstart)
def save_data(self): if os.path.exists(self.output_dir) == False: os.mkdir(self.output_dir) # generate data files and meta data file batch_num = self.batch_num batch_fname = self.batch_fname img_height = self.img_height img_width = self.img_width img_channel = self.img_channel pixelNum = img_height * img_width imgSize = pixelNum * img_channel matlab_meta = scipy.io.loadmat(self.input_matlab_meta_data_file) synsets = matlab_meta['synsets'] # costMat = matlab_meta['cost_matrix'] synsetsN = len(synsets) print "synsets length:%d" % len(synsets) wnID_2_ILSVRSID = {}; label_2_labelname = [None] * len(synsets) for i in range(len(synsets)): wnID_2_ILSVRSID[str(synsets[i][0][1][0])] = synsets[i][0][0][0][0] label_2_labelname[i] = str(synsets[i][0][2][0]) # print "wnID_2_ILSVRSID" # print wnID_2_ILSVRSID wnID = [f for f in os.listdir(self.train_dir) if os.path.isdir(os.path.join(self.train_dir, f))] wnNum = len(wnID) classSz = np.zeros((wnNum), dtype=np.uint32) # classes at leaf nodes of semantic tree wnID2label = {} for i in range(wnNum): label = -1; for j in range(synsetsN): if str(synsets[j][0][1][0]) == wnID[i]: label = j # label starts from 0 break if label < 0: print "wnID %s is not found in synsets " % wnID[i] raise ecp.NameError('error') wnID2label[wnID[i]] = label classSz[i] = synsets[label][0][7][0][0] totalImgN = sum(classSz) print 'totally %d images' % totalImgN # need to sample a subset of images because main memory can not hold all pixels in training images PCA_img_num = self.PCA_img_num if PCA_img_num > totalImgN: PCA_img_num = totalImgN print "use %d images to compute pixelwise PCA" % PCA_img_num imgID2Name = [None] * totalImgN imgID2Label = np.zeros((totalImgN), dtype=np.float32) classMember = [None] * wnNum num_image_correct = 1 imgIdx = 0 maxLabel = -1 for i in range(wnNum): label = wnID2label[wnID[i]]; if label > maxLabel: maxLabel = label subDir = wnID[i] subDirPath = os.path.join(self.train_dir, subDir) imgs = [img for img in os.listdir(subDirPath) if os.path.isfile(os.path.join(subDirPath, img)) and fnmatch.fnmatch(img, self.train_pattern)] if classSz[i] != len(imgs): print "wnID:%s %d images in folder and %d images from synset" % (wnID[i], len(imgs), classSz[i]) num_image_correct = 0 lClsMember = []; for j in range(len(imgs)): imgID2Name[imgIdx] = os.path.join(subDirPath, imgs[j]) imgID2Label[imgIdx] = label lClsMember += [imgIdx] imgIdx = imgIdx + 1 classMember[i] = lClsMember print 'max label : %d' % maxLabel if num_image_correct == 0: raise ecp.NameError('incorrect training images number') batch_size = np.zeros((wnNum, batch_num), dtype=np.uint32) for i in range(wnNum): batch_size[i][:] = get_batch_size(classSz[i], batch_num) # randomly distribute images to batches in a class-wise manner for i in range(len(classMember)): rd.shuffle(classMember[i]) # dataMean has shape (img_channel,img_height,img_width) dataMean = np.zeros((img_channel, img_height, img_width), dtype=np.float64) # dataMeanView has shape (img_height, img_width, img_channel) dataMeanView = dataMean.swapaxes(0, 2).swapaxes(0, 1) for i in range(batch_num): batchFileFn = batch_fname + '_' + str(i + self.batch_start_index) batchFileFn = os.path.join(self.output_dir, batchFileFn) print "%d th batch file name:%s" % (i, batchFileFn) batchImgN = np.sum(batch_size[:, i]) print "%d images in %d th batch out of %d batches" % (batchImgN, i, batch_num) for i in range(batch_num): batchFileFn = batch_fname + '_' + str(i + self.batch_start_index) batchFileFn = os.path.join(self.output_dir, batchFileFn) print "%d th batch file name:%s" % (i, batchFileFn) bstart = time.clock() batchImgN = np.sum(batch_size[:, i]) print "%d images in %d th batch out of %d batches" % (batchImgN, i, batch_num) # On exit, batchData has shape (imgSize, batchImgN) batchData = np.zeros((img_channel, img_height, img_width, batchImgN), dtype=np.uint8) # batchDataView shape (batchImgN, img_height, img_width, img_channel) batchDataView = np.swapaxes(batchData, 0, 3) batchLabel = np.zeros(batchImgN, dtype=np.float32) batchImgId = [] # batchImgIdx = 0 # collect image ids in the batch for j in range(wnNum): batchClsN = batch_size[j, i] startIdx = np.sum(batch_size[j, 0:i]) batchImgId += classMember[j][startIdx:startIdx + batchClsN] assert len(batchImgId) == batchImgN # important! shuffle images in the batch so that mini-batch has random class distribution rd.shuffle(batchImgId) for j in range(batchImgN): lImgID = batchImgId[j] try: im = misc.imread(imgID2Name[lImgID]) except Exception, e: print e sys.exit() if im.shape[1] != img_width or im.shape[0] != img_height: raise ecp.NameError('incorrect image size') if im.shape[2] == 3: if img_channel != 3: raise ecp.NameError('incorrect image color channels') batchDataView[j][:][:][:] = im dataMeanView += im elif im.mode == "L": print "a grey image" for p in range(img_channel): batchDataView[j][:][:][p] = im dataMeanView[:][:][p] += im else: raise ecp.NameError('non-RGB non-gray images are not handled') batchLabel[j] = imgID2Label[lImgID] # for j in range(wnNum): # if j % 200 == 0: # print "progress : %5.4f " % (float(100 * j) / float(wnNum)) # # if i < batch_num - 1: # # batchClsN = batchSize[j] # # else: # # batchClsN = lastBatchSize[j] # batchClsN = batch_size[j, i] # # # print "batchClsN %d" % batchClsN # startIdx = np.sum(batch_size[j, 0:i]) # for k in range(batchClsN): # # lImgID = classMember[j][i * batchSize[j] + k] # lImgID = classMember[j][startIdx + k] # try: # im = misc.imread(imgID2Name[lImgID]) # except Exception, e: # print e # sys.exit() # # typically, im.shape=(imgH,imgW,imgChannel) # if im.shape[1] != img_width or im.shape[0] != img_height: # raise ecp.NameError('incorrect image size') # if im.shape[2] == 3: # if img_channel != 3: # raise ecp.NameError('incorrect image color channels') # batchDataView[batchImgIdx][:][:][:] = im # dataMeanView += im # elif im.mode == "L": # print "a grey image" # for p in range(img_channel): # batchDataView[batchImgIdx][:][:][p] = im # dataMeanView[:][:][p] += im # else: # raise ecp.NameError('non-RGB non-gray images are not handled') # # batchLabel[batchImgIdx] = imgID2Label[lImgID] # batchImgIdx += 1 batchData = np.reshape(batchData, (imgSize, batchImgN)) batch_dic = {} batch_dic['data'] = batchData batch_dic['labels'] = batchLabel try: pickle(batchFileFn, batch_dic,True); except Exception as inst: print "exception occurs" print type(inst) print inst bend = time.clock() print 'batch elapsed time:%f' % (bend - bstart)
def __init__(self, dirPath, obsNames=None, create=False, refDirPath=None, name=None, ppExePath=None, ppOutputFile='observations.nc', runTime=None, runCode=None,# options for creating new study update=False, # options for updating existing study verbose=False, **parameters): """ Create an instance of HadCM3 class. Default behaviour is to read from dirPath and prohibit updates. :param dirPath -- path to directory where model simulation exists or is to be created :param create (optional with default False). If True create new directory and populate it. Afterwards the ModelSimulation will be readOnly. These options should be specified when creating a new study otherwise they are optional and ignored :param refDirPath -- reference directory. Copy all files from here into dirPath :param name -- name of the model simulation. If not provided will be taken from dirPath :param ppExePath -- path to post proessing executable :param ppOutputFile -- File name of output of post processing executable. Default is observations.nc :param obsNames -- list of observations to be readin. (see readObs()) :param runTime -- run time in seconds for UM job. If set to None nothing is changed. :param runCode -- code to be used by Job. :param update -- allow updates to the simulation information. :param verbose -- provide verbose output. (See individual methods). Default is False. : kwargs -- these are parameter and values. :returns initialised object. """ # no parameters should be provided unless create or update provided if len(parameters) >0 and not (create or update): raise exceptions.ValueError("Provided parameters but not specified create or update") # do HadCM3 specific cross checks. # should not specifiy ASTART & AINITIAL if 'AINITIAL' in parameters and 'ASTART' in parameters: raise exceptions.NameError("DO not specify both AINITIAL and ASTART.") # and OINITIAL and OSTART if 'OINITIAL' in parameters and 'OSTART' in parameters: raise exceptions.NameError("DO not specify both OINITIAL and OSTART.") # call superclass init super(HadCM3, self).__init__(dirPath, obsNames=obsNames, create=create, refDirPath=refDirPath, name=name, ppExePath=ppExePath, ppOutputFile=ppOutputFile, parameters=parameters, # options for creating new study update=update, # options for updating existing study verbose=verbose) if create: # want to create model instance so do creation. self.fixClimFCG() # fix the ClimFGC self.modifySubmit(runTime=runTime, runCode=runCode) # modify the Submit script self.modifyScript() # modify the script self.createWorkDir(refDirPath) # create the work dirctory (and fill it in) ## Set up namelist mappings # easy case all variables in SLBC21 and which just set the values. for var in ['VF1','ICE_SIZE', 'ENTCOEF','CT', 'ASYM_LAMBDA', 'CHARNOCK','G0', 'Z0FSEA']: self.simpleNamelist(var) # Hard case ones where we have a function to run or perturb multiple variables are more complex. # for meta-parameter we register function. Functions should return dict indexed by namelist info. # I think these should bs class methods to save computing them every time we create a model. # but overhead is likely small # TODO: Convert MetaFn and Namelists to class methods. self.registerMetaFn('RUNID', runName, verbose=verbose) # runid self.registerMetaFn('KAY_GWAVE', gravityWave, verbose=verbose) # gravity wave self.registerMetaFn('ALPHAM',iceAlbedo, verbose=verbose) # ice albedo self.registerMetaFn('CW_LAND',cloudWater,verbose=verbose) # CW_LAND self.registerMetaFn('RHCRIT',cloudRHcrit,verbose=verbose) # RHCRIT meta-param generates array self.registerMetaFn('EACF', cloudEACF, verbose=verbose) # EACF meta-param generates array self.registerMetaFn('DYNDIFF', diffusion, verbose=verbose) # Dynamics Diffusion generates lots of arrays self.registerMetaFn('RUN_TARGET',runTarget,verbose=verbose) # length of simulation -- modifies several namelist vars self.registerMetaFn('START_TIME', startTime, verbose=verbose) # start_Time for run -- modifies several namelist vars self.registerMetaFn("SPHERICAL_ICE",sph_ice,verbose=verbose) # Spherical ice (or not) # add AINITIAL, OINITIAL, ASTART & OSTART in inithist for var in ['AINITIAL', 'OINITIAL', 'ASTART', 'OSTART']: # need slightly special code for these variables. # ideas tis to use the same basic function which has as argument the parameter. Then use functools.partial to # generate function which gets registered. fn=functools.partial(initHist_nlcfiles, parameter=var) fn.func_name=var.lower()+'_initHist_nlcfiles' self.registerMetaFn(var, fn, verbose=verbose) # got some parameters and either creating or updating -- update namelist. if len(parameters) >0 and (create or update): self.setReadOnly(False) # allow modification self.setParams(parameters,verbose=verbose,fail=True) # apply namelist etc # deal with START_TIME -- f90nml.patch fails... nad f90nml.read doesn't work as it should # need a temp files etc -- TODO: If need this functionality again then warp in a subroutine and try and fix f90nml! # if 'START_TIME' in parameters: RECONA_FILE=os.path.join(self.dirPath,'RECONA') recona=f90nml.read(RECONA_FILE) if f90nml.__version__ != '0.21': # version should be 0.21 raise Exception("Can only work with f90nml version 0.21. Version is %s"%(f90nml.__version__)) # f90nml is .20 then it appears to truncate the array # so fill it in, # general problem here is that array length varies in rather arbitrary ways. # the only obvious reference point we have is 405 at fixhs[11] -- which means vn4.5 TODO fix this. fixhd=recona['headers']['FIXHD'] # f90nml (even at vn 0.21) has problems reading in nml in the following format # fixhd(12)=xxx -- instead putting the value at posn 0. offset=fixhd.index(405)-11 # if 405 not found will trigger an error. print "patching fixhd in %s is "%RECONA_FILE,repr(fixhd) #all this is done because f90nml seems to remember it starts at some posn. fixhd[20+offset]=parameters['START_TIME'][0] fixhd[27+offset]=parameters['START_TIME'][0] recona.write(RECONA_FILE, force=True) # overwrite the existing file self.setReadOnly(True) # make it read only