def __init__(self, name): self.config_file_name = name try: doc = parse(self.config_file_name) except IOError: print '****** Could not find configure file!' raise IOError # default value self.datarootdir = "/data1/sensors/atrain/cloudsat/CLDCLASS/R04" ###self.datarootdir = "/data1/sensors/atrain/amsre.aqua" elems = doc.getElementsByTagName('data_root_dir') ###print 'elems type: ', type(elems[0]) ###print 'elems: ', elems[0].childNodes if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: ###print 'c: ', c.data self.datarootdir = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing data_root_dir in config file" print " config filename: ", self.config_file_name print " using : ", self.datarootdir print 'self.datarootdir: ', self.datarootdir # get front end name, if present in xml file # gblock 3/17/2011 - added front end name to xml file self.dataset_name = '' elems = doc.getElementsByTagName('dataset_name') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: ###print 'c: ', c.data self.dataset_name = (c.data).encode('UTF-8').strip('\n').strip() # check for name defined by xml statement if (self.dataset_name == ''): self.dataset_name = UT.parse_data_type (self.datarootdir) print 'self.dataset_name: ', self.dataset_name # get container for dataset self.dataset_container = dataset_registry.get_dataset_container (self.dataset_name) self.dataset_container.set_datarootdir (self.datarootdir) # default value self.listdir = os.path.expanduser('~') elems = doc.getElementsByTagName('list_dir') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.listdir = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing list_dir in config file" print " config filename: ", self.config_file_name print " using : ", self.listdir ###print 'self.listdir: ', self.listdir # default value self.outputdir = os.path.expanduser('~') elems = doc.getElementsByTagName('output_dir') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.outputdir = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing output_dir in config file" print " config filename: ", self.config_file_name print " using : ", self.outputdir print 'self.outputdir: ', self.outputdir # default value self.start_t = datetime.strptime('11/02/2006 03:15:25', "%m/%d/%Y %H:%M:%S") # TBD: use CloudSat start time elems = doc.getElementsByTagName('start_time') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: st = (c.data).encode('UTF-8').strip('\n').strip() ###print 'st: ', st ### self.start_t = datetime.strptime(st, "%m/%d/%Y %H:%M:%S%p") self.start_t = datetime.strptime(st, "%m/%d/%Y %H:%M:%S") else: print "***** warning in file_list.__init__ - missing start_time in config file" print " config filename: ", self.config_file_name print " using : ", self.start_t # default value self.end_t = datetime.strptime('11/04/2006 07:35:20', "%m/%d/%Y %H:%M:%S") # TBD: use CloudSat end time elems = doc.getElementsByTagName('end_time') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: et = (c.data).encode('UTF-8').strip('\n').strip() ###print 'et: ', et self.end_t = datetime.strptime(et, "%m/%d/%Y %H:%M:%S") else: print "***** warning in file_list.__init__ - missing end_time in config file" print " config filename: ", self.config_file_name print " using : ", self.end_t # default value (2min) self.time_diff = '120' elems = doc.getElementsByTagName('time_diff') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.time_diff = (c.data).encode('UTF-8').strip('\n').strip() ### print 'self.time_diff: ', self.time_diff else: print "***** warning in file_list.__init__ - missing time_diff in config file" print " config filename: ", self.config_file_name print " using : ", self.time_diff # default value (15min) self.time_search_range = '900' elems = doc.getElementsByTagName('time_search_range') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.time_search_range = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing time_search_range in config file" print " config filename: ", self.config_file_name print " using : ", self.time_search_range print 'self.time_search_range: ', self.time_search_range # default value (20km) self.footprint_size = '20' elems = doc.getElementsByTagName('footprint_size') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.footprint_size = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing footprit_size in config file" print " config filename: ", self.config_file_name print " using : ", self.footprint_size print 'self.footprint_size: ', self.footprint_size # default value (1.5) self.space_search_factor = '1.5' elems = doc.getElementsByTagName('space_search_factor') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.space_search_factor = (c.data).encode('UTF-8').strip('\n').strip() else: print "***** warning in file_list.__init__ - missing space_search_factor in config file" print " config filename: ", self.config_file_name print " using : ", self.space_search_factor print 'self.space_search_factor: ', self.space_search_factor # default value (40) self.cell_search_limit = '40' elems = doc.getElementsByTagName('cell_search_limit') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.cell_search_limit = (c.data).encode('UTF-8').strip('\n').strip() ###else: ###print "***** warning in file_list.__init__ - missing cell_search_limit in config file" ###print " config filename: ", self.config_file_name ###print " using : ", self.cell_search_limit print 'self.cell_search_limit: ', self.cell_search_limit # default value (-909090) self.invalid_data = '-909090' elems = doc.getElementsByTagName('invalid_data') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.invalid_data = (c.data).encode('UTF-8').strip('\n').strip() print 'self.invalid_data: ', self.invalid_data # default value ('None') self.missing_value = 'None' elems = doc.getElementsByTagName('missing_value') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.missing_value = (c.data).encode('UTF-8').strip('\n').strip() print 'self.missing_value: ', self.missing_value # default value (1) self.num_cores = '1' elems = doc.getElementsByTagName('num_cores') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.num_cores = (c.data).encode('UTF-8').strip('\n').strip() print 'self.num_cores: ', self.num_cores # default value (1) self.proc_size = '1' elems = doc.getElementsByTagName('proc_size') if len(elems) > 0: for c in elems[0].childNodes: if c.nodeType != Node.COMMENT_NODE and len(c.data) > 1: self.proc_size = (c.data).encode('UTF-8').strip('\n').strip() print 'self.proc_size: ', self.proc_size # set from list, for fast membership check self.dirNameSet = set(self.dirNameList)
import util as UT src_data = UT.parse_data_type('/mnt/nas-0-0/users/collocation/ecmwf/idaily-surface') src_file = '/mnt/nas-0-0/users/collocation/ecmwf/idaily-surface/2008/01/ecmwf-interim-daily-surface-2008-01-06.nc' src_front_end = UT.get_front_end(src_data, src_file) #### print 'src_front_end: ', src_front_end print 'time: ', src_front_end.get_time() print 'lat: ', src_front_end.get_latitude() print 'lon: ', src_front_end.get_longitude() print 'data: ', src_front_end.get_data() print 'grid info: ', src_front_end.get_src_uniform_grid_info()
t2 = afl1.startTimeList[index2] f2 = afl1.dirNameList[index2] if index2 < index1: print '****** Error: In user specified time range, no target data set exists!' sys.exit(-1) print target_data + ' (start) time range:', t1, t2 print target_data + ' file range:', f1, f2 print 'User specified ' + target_data + ' granule number range:', afl1.granuleNumList[ index1], afl1.granuleNumList[index2] # find out what data set the source data is src_data = UT.parse_data_type(afl.datarootdir) print 'src_data: ', src_data # load the file list for src data granules (afl.listdir is where all the list files reside) list_file1 = afl.listdir + '/' + src_data + '_granules.list.pkl' print 'list_file1: ', list_file1 try: afl.load_lists(list_file1) except IOError: print '****** list file for ' + src_data + ' does not exist.' sys.exit(-1) # Instantiate a middle end class midEnd = MD.middle_end()
index2 = len(afl1.startTimeList) - 1 t2 = afl1.startTimeList[index2] f2 = afl1.dirNameList[index2] if index2 < index1: print '****** Error: In user specified time range, no target data set exists!' sys.exit(-1) print target_data+' (start) time range:', t1, t2 print target_data+' file range:', f1, f2 print 'User specified '+target_data+' granule number range:', afl1.granuleNumList[index1], afl1.granuleNumList[index2] # find out what data set the source data is src_data = UT.parse_data_type(afl.datarootdir) print 'src_data: ', src_data # load the file list for src data granules (afl.listdir is where all the list files reside) list_file1 = afl.listdir + '/' + src_data + '_granules.list.pkl' print 'list_file1: ', list_file1 try: afl.load_lists(list_file1) except IOError: print '****** list file for ' + src_data + ' does not exist.' sys.exit(-1) # Instantiate a middle end class midEnd = MD.middle_end()
import util as UT src_data = UT.parse_data_type( '/mnt/nas-0-0/users/collocation/ecmwf/idaily-surface') src_file = '/mnt/nas-0-0/users/collocation/ecmwf/idaily-surface/2008/01/ecmwf-interim-daily-surface-2008-01-06.nc' src_front_end = UT.get_front_end(src_data, src_file) #### print 'src_front_end: ', src_front_end print 'time: ', src_front_end.get_time() print 'lat: ', src_front_end.get_latitude() print 'lon: ', src_front_end.get_longitude() print 'data: ', src_front_end.get_data() print 'grid info: ', src_front_end.get_src_uniform_grid_info()