def insert_lists(self, full_file_name, file_name): #gblock start_time = UT.get_start_time(full_file_name, file_name) #gblock end_time = UT.get_end_time(full_file_name, file_name) start_time = self.dataset_container.get_start_time(file_name) end_time = self.dataset_container.get_end_time(file_name) # Keep inserting start time (in unix time format) into a running list of # such start times. The insertion point into this list determines the insertion # point for the other (parallel) lists that support cataloging. These other # lists include a filename lists and lists for start times and end time in # datetime format itime = UT.datetime_to_unix_time(start_time) pos = bisect.bisect(self.uTimeList, itime) self.uTimeList.insert(pos, itime) # update the list of filenames in the same manner self.dirNameList.insert(pos, full_file_name) # update the list of start times in the same manner self.startTimeList.insert(pos, start_time) # update the list of end time in the same manner self.endTimeList.insert(pos, end_time) # maintan set for fast membership check self.dirNameSet.add(full_file_name) # Cloudsat has one more list #print full_file_name if(full_file_name.find('cloudsat') >= 0): self.granuleNumList.insert(pos, file_name[14:19]) # CloudSat granule number if(full_file_name.find('H2O') >= 0): self.granuleNumList.insert(pos, file_name[29:37]) # CloudSat granule number
def validate_front_end (dset, rootdir, filename): # get start and end times short_start = filename.rfind ('/')+1 short_name = filename[short_start:] #print short_start, short_name start_time_d = dset.get_start_time (short_name) end_time_d = dset.get_end_time (short_name) start_time_u = util.datetime_to_unix_time (start_time_d) end_time_u = util.datetime_to_unix_time (end_time_d) # set root directory dset.set_datarootdir (rootdir) # get front end for dataset cs = dset.get_front_end (filename) # try all required interfaces cs.create_catalog() print "catalog created" grid_info = cs.get_src_uniform_grid_info() grid_size = grid_info[0] * grid_info[1] * grid_info[2] print "grid info = ", grid_info, grid_size last = grid_size-1 # get temporal and spatial information time = cs.get_time() lat = cs.get_latitude() long = cs.get_longitude() # first dimension is always indexed spatial/temproral datapoint number DptCnt = time.shape[0] # validate time #print "time = ", time[0:9], time[last] prev_time = time[0] for TimeNo in range (0, DptCnt): cur_time = time[TimeNo] # check time between start and end times if (cur_time < start_time_u) or (cur_time > end_time_u): print "error in validate_front_end - current time out of range" print "indx: ", TimeNo, ", Val: ", cur_time print "start time: ", start_time_u, ", end time: ", end_time_u # check time same or incrasing if cur_time < prev_time: print "error in validate_front_end - current time decreasing" print "indx: ", TimeNo, ", cur time: ", cur_time, ", prev time:", prev_time # validate latitude #print "latitude = ", lat[0:9], lat[last] # first dimension must be same size as count of spatial/temporal datapoints if lat.shape[0] != DptCnt: print "error in dim 0 of latitude - size of dimension does not match spatial/temproral dim size" print "Lat size:", Lat.shape[0], ", Dpt size:", DptCnt for LatNo in range(0, DptCnt): if (lat[LatNo] <= -90.0) or (lat[LatNo] >= 90.0): print "**** Error in get_latitude - lat value out of range" print "indx: ", LatNo, ", val:", lat[LatNo] # validate longitude #print "longitude = ", long[0:9], long[last] # first dimension must be same size as count of spatial/temporal datapoints LonCnt = long.shape[0] if LonCnt != DptCnt: print "error in dim 0 of longitude - size of dimension does not match spatial/temproral dim size" print "Lon size:", LonCnt, ", Dpt size:", DptCnt for LongNo in range(0, DptCnt): if (long[LongNo] < -180.0) or (long[LongNo] > 180.0): print "**** Error in get_longitude - long value out of range" print "indx: ", LongNo, ", val:", long[LongNo] levels = cs.get_levels() for vname in levels.keys(): print "level ", vname var = levels[vname] attr = var[0] data = var[1] for attrName in attr.keys(): attrVal = attr[attrName] print ' attr[', attrName, '] = ', str(attrVal) print "data = ", data dataset = cs.get_data() #print dataset.keys() for vname in dataset.keys(): var = dataset[vname] attr = var[0] data = var[1] print 'var ', vname, data.shape # first dimension must be same size as count of spatial/temporal datapoints if data.shape[0] != DptCnt: print "error in dim 0 of " + vname + \ " - size of dimension does not match spatial/temproral dim size" print "Dim 0 size:", data.shape[0], ", Dpt size:", DptCnt # test for level data (has dimension1 attribute) # if so, must match level shape # display attributes for attrName in attr.keys(): attrVal = attr[attrName] print ' attr[', attrName, '] = ', str(attrVal) # check for required attributes """ # display summary according to shape rank = len(data.shape) if rank == 1: print "data = ", data[0:9], data[last] elif rank == 2: print "data = ", data[0:9,0], data[last,0] elif rank == 3: print "data = ", data[0:9,0,0], data[last,0,0] else: print "rank: ", rank """ print "min:", N.min(data), "max:", N.max(data)
def validate_front_end(dset, rootdir, filename): # get start and end times short_start = filename.rfind('/') + 1 short_name = filename[short_start:] #print short_start, short_name start_time_d = dset.get_start_time(short_name) end_time_d = dset.get_end_time(short_name) start_time_u = util.datetime_to_unix_time(start_time_d) end_time_u = util.datetime_to_unix_time(end_time_d) # set root directory dset.set_datarootdir(rootdir) # get front end for dataset cs = dset.get_front_end(filename) # try all required interfaces cs.create_catalog() print "catalog created" grid_info = cs.get_src_uniform_grid_info() grid_size = grid_info[0] * grid_info[1] * grid_info[2] print "grid info = ", grid_info, grid_size last = grid_size - 1 # get temporal and spatial information time = cs.get_time() lat = cs.get_latitude() long = cs.get_longitude() # first dimension is always indexed spatial/temproral datapoint number DptCnt = time.shape[0] # validate time #print "time = ", time[0:9], time[last] prev_time = time[0] for TimeNo in range(0, DptCnt): cur_time = time[TimeNo] # check time between start and end times if (cur_time < start_time_u) or (cur_time > end_time_u): print "error in validate_front_end - current time out of range" print "indx: ", TimeNo, ", Val: ", cur_time print "start time: ", start_time_u, ", end time: ", end_time_u # check time same or incrasing if cur_time < prev_time: print "error in validate_front_end - current time decreasing" print "indx: ", TimeNo, ", cur time: ", cur_time, ", prev time:", prev_time # validate latitude #print "latitude = ", lat[0:9], lat[last] # first dimension must be same size as count of spatial/temporal datapoints if lat.shape[0] != DptCnt: print "error in dim 0 of latitude - size of dimension does not match spatial/temproral dim size" print "Lat size:", Lat.shape[0], ", Dpt size:", DptCnt for LatNo in range(0, DptCnt): if (lat[LatNo] <= -90.0) or (lat[LatNo] >= 90.0): print "**** Error in get_latitude - lat value out of range" print "indx: ", LatNo, ", val:", lat[LatNo] # validate longitude #print "longitude = ", long[0:9], long[last] # first dimension must be same size as count of spatial/temporal datapoints LonCnt = long.shape[0] if LonCnt != DptCnt: print "error in dim 0 of longitude - size of dimension does not match spatial/temproral dim size" print "Lon size:", LonCnt, ", Dpt size:", DptCnt for LongNo in range(0, DptCnt): if (long[LongNo] < -180.0) or (long[LongNo] > 180.0): print "**** Error in get_longitude - long value out of range" print "indx: ", LongNo, ", val:", long[LongNo] levels = cs.get_levels() for vname in levels.keys(): print "level ", vname var = levels[vname] attr = var[0] data = var[1] for attrName in attr.keys(): attrVal = attr[attrName] print ' attr[', attrName, '] = ', str(attrVal) print "data = ", data dataset = cs.get_data() #print dataset.keys() for vname in dataset.keys(): var = dataset[vname] attr = var[0] data = var[1] print 'var ', vname, data.shape # first dimension must be same size as count of spatial/temporal datapoints if data.shape[0] != DptCnt: print "error in dim 0 of " + vname + \ " - size of dimension does not match spatial/temproral dim size" print "Dim 0 size:", data.shape[0], ", Dpt size:", DptCnt # test for level data (has dimension1 attribute) # if so, must match level shape # display attributes for attrName in attr.keys(): attrVal = attr[attrName] print ' attr[', attrName, '] = ', str(attrVal) # check for required attributes """ # display summary according to shape rank = len(data.shape) if rank == 1: print "data = ", data[0:9], data[last] elif rank == 2: print "data = ", data[0:9,0], data[last,0] elif rank == 3: print "data = ", data[0:9,0,0], data[last,0,0] else: print "rank: ", rank """ print "min:", N.min(data), "max:", N.max(data)