def run(self): c_code = self.read_c_code() c_code = self.pre_process(c_code) m = middle_end(c_code) self.asm = self.asm + m.run() + self.global_decl self.write_assembly() b = jz_101_assembler() b.run(self.asm) self.write_hex(b) self.write_secondary_output(b)
def sdriver(gid, pid, src_data, target_data, afl, afl1, index1, index2): print '--------- inside sdriver(), gid: ', gid, ', pid: ', pid, ' ----------' print 'index1: ', index1, ', index2: ', index2 start = datetime.datetime.now() # get front end dataset container (gblock 3/21/2011) dset_container = afl.dataset_container # Instantiate a middle end class midEnd = MD.middle_end() # Instantiate a back end class bakEnd = BD.back_end('netCDF') # keeps track of the indices from the previous loop pre_idx1 = -1 pre_idx2 = -1 # Makes sure we only create the supported variables report # per run # (moved to pdriver.py) """ catalog_created = 0 """ # save these values from last i iteration, just in case # in the current i loop, there is no need to read from # any j loop src granules num_time_steps0 = 0 num_lat0 = 0 num_lon0 = 0 src_front_end = None # loop over the sub- file list in target data set for i in range(index1, index2+1): print '' print '' print '' print '' # for each file, read in data tgt_file = afl1.dirNameList[i] # Construct target data structure in the front end and read in the data print "****** Reading target data from file: ", tgt_file print 'granule num:', afl1.granuleNumList[i] print 'file start/end time:', afl1.startTimeList[i], afl1.endTimeList[i] if target_data == 'cloudsat': tgt_front_end = FEC.front_end_cloudsat(tgt_file) elif target_data == 'mls-h2o': import front_end_mls_h2o tgt_front_end = front_end_mls_h2o.front_end_mls_h2o(tgt_file) else: print '****** Error: '+target_data+' centric co-location is not supported at this moment !' sys.exit(-1) # Pass the target grid info to the middle end grid data structure # Get time info as an array midEnd.set_target_time(tgt_front_end.get_time()) print 'target Unix Time = ', midEnd.get_target_time() print 'target Unix Time Range = ', min(midEnd.get_target_time()), max(midEnd.get_target_time()) print 'target Unix Time Size = ', midEnd.get_target_time().size print '' # Get latitude info as an array midEnd.set_target_lat(tgt_front_end.get_latitude()) print 'target Latitude = ', midEnd.get_target_lat() print 'target Latitude Range = ', min(midEnd.get_target_lat()), max(midEnd.get_target_lat()) print 'target Latitude Size = ', midEnd.get_target_lat().size print '' # Get longitude info as an array midEnd.set_target_lon(tgt_front_end.get_longitude()) print 'target Longitude = ', midEnd.get_target_lon() print 'target Longitude Range = ', min(midEnd.get_target_lon()), max(midEnd.get_target_lon()) print 'target Longitude Size = ', midEnd.get_target_lon().size print "****** target data acquired! ******" print '' print '' # for each target granule, find time range st1 = afl1.startTimeList[i] et1 = afl1.endTimeList[i] ###print "****** src granule start time before adjustment:", st1 # adjust start time (some 3min earlier becaue src is leading target) st1 -= datetime.timedelta(seconds=int(afl.time_diff)) # adjust search range st1 -= datetime.timedelta(seconds=int(afl.time_search_range)) print "****** target granule start time after adjustment:", st1 # adjust end time (some 3min earlier becaue src is leading target) et1 -= datetime.timedelta(seconds=int(afl.time_diff)) # adjust search range et1 += datetime.timedelta(seconds=int(afl.time_search_range)) print "****** target granule end time after adjustment:", et1 # search for src sub- file list that fall in time range idx1 = bisect.bisect_left(afl.endTimeList, st1) print 'idx1: ', idx1 if idx1 == len(afl.endTimeList): idx1 -= 1 print '****** Warning: search result indx1 is at the end of afl1.endTimeList' print 'idx1 (after shift): ', idx1 ### print 'len(afl.endTimeList): ', len(afl.endTimeList) ### print 'max(afl.endTimeList): ', max(afl.endTimeList), ' min(afl.endTimeList): ', min(afl.endTimeList) tt1 = afl.startTimeList[idx1] ff1 = afl.dirNameList[idx1] # end time idx2 = bisect.bisect_right(afl.startTimeList, et1) - 1 print 'idx2: ', idx2 if idx2 == len(afl.startTimeList): idx2 -= 1 print '****** Warning: search result indx2 is at the end of afl1.startTimeList' print 'idx2 (after shift): ', idx2 tt2 = afl.startTimeList[idx2] ff2 = afl.dirNameList[idx2] # check tt2 and tt1. skip to the next i loop if # no src granule is found for the target granule if tt1 > tt2: print '****** Warning: there is no source granule within the target time search range.' print ' Skip to the next target granule.' continue print 'st1: ', st1, ' et1: ', et1, 'idx1: ', idx1, ' idx2: ', idx2 print 'src (start) time range:', tt1, tt2 print 'src file range:', ff1, ff2 # do not read any src file that's already read if idx1 <= pre_idx2: idx1 = pre_idx2 + 1 if idx1 > idx2: print "****** No need to read src data from file because it's been already read." print 'idx1: ', idx1, ', idx2: ', idx2 start_reading_src_time = datetime.datetime.now() # loop over src files for j in range(idx1, idx2+1): print '' # read in data and co-locate to target grid print 'loop over src files j: ', j print 'src file:', afl.dirNameList[j] src_file = afl.dirNameList[j] # Read src data print "****** Reading src data from file: ", src_file # initialize a front end (gblock, 3/22/2011) src_front_end = dset_container.get_front_end (src_file) print 'src_front_end: ', src_front_end # Create a report of variables supported # (moved to pdriver.py) """ if catalog_created != 1: src_front_end.create_catalog() catalog_created = 1 """ midEnd.set_src_time(src_front_end.get_time()) print 'src Unix Time size = ', midEnd.get_src_time().size print 'src Unix Time = ', midEnd.get_src_time() print 'src Unix Time Range = ', min(midEnd.get_src_time()), max(midEnd.get_src_time()) #zzzz if j>idx1: temp1 = midEnd.src_time[1:] - midEnd.src_time[:-1] print 'src_time, Is it sortedsorted?' print temp1.min() print temp1.max() print midEnd.src_time.shape if temp1.min()<0.0: import numpy as np np.save('/home/bytang/projects/collocation/output/src_time_%02d.npy'%(j), \ midEnd.src_time) midEnd.set_src_lat(src_front_end.get_latitude()) print 'src Latitude size = ', midEnd.get_src_lat().size print 'src Latitude Range = ', min(midEnd.get_src_lat()), max(midEnd.get_src_lat()) print 'src Latitude = ', midEnd.get_src_lat() midEnd.set_src_lon(src_front_end.get_longitude()) print 'src Longitude Range = ', min(midEnd.get_src_lon()), max(midEnd.get_src_lon()) print 'src Longitude size = ', midEnd.get_src_lon().size print 'src Longitude = ', midEnd.get_src_lon() start1 = datetime.datetime.now() midEnd.set_src_data(src_front_end.get_data()) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print 'elapsed_time: ', elapsed_time print '*** midEnd.set_src_data elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print "****** src data acquired! ******" # end of for j loop (over src granules) # for model data, tell the middle end about the uniform grid info. # so time interpolation can be done # # special note: complex "if" statment has been moved to dset_nemo.py # new front ends should use new dset containers & register container in util.py # see dset_ecmwf_idaily_surface_ncar.py for example # end_reading_src_time = datetime.datetime.now() elapsed_time = end_reading_src_time - start_reading_src_time print '*** src reading and assigning elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' if dset_container.is_uniform_grid (): print 'sdriver (dataset is on uniform grid) - src_front_end: ', src_front_end print 'using time interpolation for co-location' if src_front_end == None: num_time_steps = num_time_steps0 num_lat = num_lat0 num_lon = num_lon0 else: (num_time_steps, num_lat, num_lon) = src_front_end.get_src_uniform_grid_info() # save the values for reuse possibly in the next i loop num_time_steps0 = num_time_steps num_lat0 = num_lat num_lon0 = num_lon midEnd.set_src_uniform_grid_info(num_time_steps, num_lat, num_lon) start1 = datetime.datetime.now() midEnd.co_locate(afl, afl1) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print '*** midEnd.co_locate elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print '****** midEnd.good_cnt: ', midEnd.good_cnt print '****** success match rate (%): ', (100.0*midEnd.good_cnt)/midEnd.get_target_time().size, 'for target granule ', afl1.granuleNumList[i] if midEnd.good_cnt == 0: print 'No co-located data. Will not call the back-end. Skip to the next target granule.' continue start1 = datetime.datetime.now() # set target file name tgt_file_name = os.path.basename(tgt_file) dp = tgt_file_name.find('.') tf_name = tgt_file_name[0:dp] # for visualization of nearest neighbors and footprints # this is not essential to the product and can be commented out ### midEnd.set_output_filename(afl.outputdir+'/'+'latlon_'+src_data+'_'+tf_name+'.txt') ### midEnd.output_latlon() # pass data to back end bakEnd.set_target_time(midEnd.get_target_time()) bakEnd.set_target_lat(midEnd.get_target_lat()) bakEnd.set_target_lon(midEnd.get_target_lon()) bakEnd.set_target_data(midEnd.get_target_data()) bakEnd.set_invalid_data(midEnd.get_invalid_data()) ### print 'before set_target_levels(), src_front_end: ', src_front_end bakEnd.set_target_levels(src_front_end.get_levels()) # write out to output file bakEnd.set_output_filename(afl.outputdir+'/'+src_data+'_'+tf_name+'.nc') bakEnd.write_output(afl) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print '*** backend elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' # To aid when remote profiling only #time.sleep(60) pre_idx1 = idx1 pre_idx2 = idx2 # end of for i loop (over target granules) now = datetime.datetime.now() elapsed_time = now - start print '*** sdriver elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print '--------- exiting sdriver(), gid: ', gid, ', pid: ', pid, ' ----------'
src_data = UT.parse_data_type(afl.datarootdir) print 'src_data: ', src_data # load the file list for src data granules (afl.listdir is where all the list files reside) list_file1 = afl.listdir + '/' + src_data + '_granules.list.pkl' print 'list_file1: ', list_file1 try: afl.load_lists(list_file1) except IOError: print '****** list file for ' + src_data + ' does not exist.' sys.exit(-1) # Instantiate a middle end class midEnd = MD.middle_end() # Instantiate a back end class bakEnd = BD.back_end('netCDF') # keeps track of the indices from the previous loop pre_idx1 = 0 pre_idx2 = 0 # Makes sure we only create the supported variables report # per run catalog_created = 0 # loop over the sub- file list in target data set for i in range(index1, index2+1): print ''
# find out what data set the source data is src_data = UT.parse_data_type(afl.datarootdir) print 'src_data: ', src_data # load the file list for src data granules (afl.listdir is where all the list files reside) list_file1 = afl.listdir + '/' + src_data + '_granules.list.pkl' print 'list_file1: ', list_file1 try: afl.load_lists(list_file1) except IOError: print '****** list file for ' + src_data + ' does not exist.' sys.exit(-1) # Instantiate a middle end class midEnd = MD.middle_end() # Instantiate a back end class bakEnd = BD.back_end('netCDF') # keeps track of the indices from the previous loop pre_idx1 = 0 pre_idx2 = 0 # Makes sure we only create the supported variables report # per run catalog_created = 0 # loop over the sub- file list in target data set for i in range(index1, index2 + 1): print ''
def sdriver(gid, pid, src_data, target_data, afl, afl1, index1, index2): print '--------- inside sdriver(), gid: ', gid, ', pid: ', pid, ' ----------' print 'index1: ', index1, ', index2: ', index2 start = datetime.datetime.now() # get front end dataset container (gblock 3/21/2011) dset_container = afl.dataset_container # Instantiate a middle end class midEnd = MD.middle_end() # Instantiate a back end class bakEnd = BD.back_end('netCDF') # keeps track of the indices from the previous loop pre_idx1 = -1 pre_idx2 = -1 # Makes sure we only create the supported variables report # per run # (moved to pdriver.py) """ catalog_created = 0 """ # save these values from last i iteration, just in case # in the current i loop, there is no need to read from # any j loop src granules num_time_steps0 = 0 num_lat0 = 0 num_lon0 = 0 src_front_end = None # loop over the sub- file list in target data set for i in range(index1, index2 + 1): print '' print '' print '' print '' # for each file, read in data tgt_file = afl1.dirNameList[i] # Construct target data structure in the front end and read in the data print "****** Reading target data from file: ", tgt_file print 'granule num:', afl1.granuleNumList[i] print 'file start/end time:', afl1.startTimeList[i], afl1.endTimeList[ i] if target_data == 'cloudsat': tgt_front_end = FEC.front_end_cloudsat(tgt_file) elif target_data == 'mls-h2o': import front_end_mls_h2o tgt_front_end = front_end_mls_h2o.front_end_mls_h2o(tgt_file) else: print '****** Error: ' + target_data + ' centric co-location is not supported at this moment !' sys.exit(-1) # Pass the target grid info to the middle end grid data structure # Get time info as an array midEnd.set_target_time(tgt_front_end.get_time()) print 'target Unix Time = ', midEnd.get_target_time() print 'target Unix Time Range = ', min(midEnd.get_target_time()), max( midEnd.get_target_time()) print 'target Unix Time Size = ', midEnd.get_target_time().size print '' # Get latitude info as an array midEnd.set_target_lat(tgt_front_end.get_latitude()) print 'target Latitude = ', midEnd.get_target_lat() print 'target Latitude Range = ', min(midEnd.get_target_lat()), max( midEnd.get_target_lat()) print 'target Latitude Size = ', midEnd.get_target_lat().size print '' # Get longitude info as an array midEnd.set_target_lon(tgt_front_end.get_longitude()) print 'target Longitude = ', midEnd.get_target_lon() print 'target Longitude Range = ', min(midEnd.get_target_lon()), max( midEnd.get_target_lon()) print 'target Longitude Size = ', midEnd.get_target_lon().size print "****** target data acquired! ******" print '' print '' # for each target granule, find time range st1 = afl1.startTimeList[i] et1 = afl1.endTimeList[i] ###print "****** src granule start time before adjustment:", st1 # adjust start time (some 3min earlier becaue src is leading target) st1 -= datetime.timedelta(seconds=int(afl.time_diff)) # adjust search range st1 -= datetime.timedelta(seconds=int(afl.time_search_range)) print "****** target granule start time after adjustment:", st1 # adjust end time (some 3min earlier becaue src is leading target) et1 -= datetime.timedelta(seconds=int(afl.time_diff)) # adjust search range et1 += datetime.timedelta(seconds=int(afl.time_search_range)) print "****** target granule end time after adjustment:", et1 # search for src sub- file list that fall in time range idx1 = bisect.bisect_left(afl.endTimeList, st1) print 'idx1: ', idx1 if idx1 == len(afl.endTimeList): idx1 -= 1 print '****** Warning: search result indx1 is at the end of afl1.endTimeList' print 'idx1 (after shift): ', idx1 ### print 'len(afl.endTimeList): ', len(afl.endTimeList) ### print 'max(afl.endTimeList): ', max(afl.endTimeList), ' min(afl.endTimeList): ', min(afl.endTimeList) tt1 = afl.startTimeList[idx1] ff1 = afl.dirNameList[idx1] # end time idx2 = bisect.bisect_right(afl.startTimeList, et1) - 1 print 'idx2: ', idx2 if idx2 == len(afl.startTimeList): idx2 -= 1 print '****** Warning: search result indx2 is at the end of afl1.startTimeList' print 'idx2 (after shift): ', idx2 tt2 = afl.startTimeList[idx2] ff2 = afl.dirNameList[idx2] # check tt2 and tt1. skip to the next i loop if # no src granule is found for the target granule if tt1 > tt2: print '****** Warning: there is no source granule within the target time search range.' print ' Skip to the next target granule.' continue print 'st1: ', st1, ' et1: ', et1, 'idx1: ', idx1, ' idx2: ', idx2 print 'src (start) time range:', tt1, tt2 print 'src file range:', ff1, ff2 # do not read any src file that's already read if idx1 <= pre_idx2: idx1 = pre_idx2 + 1 if idx1 > idx2: print "****** No need to read src data from file because it's been already read." print 'idx1: ', idx1, ', idx2: ', idx2 start_reading_src_time = datetime.datetime.now() # loop over src files for j in range(idx1, idx2 + 1): print '' # read in data and co-locate to target grid print 'loop over src files j: ', j print 'src file:', afl.dirNameList[j] src_file = afl.dirNameList[j] # Read src data print "****** Reading src data from file: ", src_file # initialize a front end (gblock, 3/22/2011) src_front_end = dset_container.get_front_end(src_file) print 'src_front_end: ', src_front_end # Create a report of variables supported # (moved to pdriver.py) """ if catalog_created != 1: src_front_end.create_catalog() catalog_created = 1 """ midEnd.set_src_time(src_front_end.get_time()) print 'src Unix Time size = ', midEnd.get_src_time().size print 'src Unix Time = ', midEnd.get_src_time() print 'src Unix Time Range = ', min(midEnd.get_src_time()), max( midEnd.get_src_time()) #zzzz if j > idx1: temp1 = midEnd.src_time[1:] - midEnd.src_time[:-1] print 'src_time, Is it sortedsorted?' print temp1.min() print temp1.max() print midEnd.src_time.shape if temp1.min() < 0.0: import numpy as np np.save('/home/bytang/projects/collocation/output/src_time_%02d.npy'%(j), \ midEnd.src_time) midEnd.set_src_lat(src_front_end.get_latitude()) print 'src Latitude size = ', midEnd.get_src_lat().size print 'src Latitude Range = ', min(midEnd.get_src_lat()), max( midEnd.get_src_lat()) print 'src Latitude = ', midEnd.get_src_lat() midEnd.set_src_lon(src_front_end.get_longitude()) print 'src Longitude Range = ', min(midEnd.get_src_lon()), max( midEnd.get_src_lon()) print 'src Longitude size = ', midEnd.get_src_lon().size print 'src Longitude = ', midEnd.get_src_lon() start1 = datetime.datetime.now() midEnd.set_src_data(src_front_end.get_data()) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print 'elapsed_time: ', elapsed_time print '*** midEnd.set_src_data elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print "****** src data acquired! ******" # end of for j loop (over src granules) # for model data, tell the middle end about the uniform grid info. # so time interpolation can be done # # special note: complex "if" statment has been moved to dset_nemo.py # new front ends should use new dset containers & register container in util.py # see dset_ecmwf_idaily_surface_ncar.py for example # end_reading_src_time = datetime.datetime.now() elapsed_time = end_reading_src_time - start_reading_src_time print '*** src reading and assigning elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' if dset_container.is_uniform_grid(): print 'sdriver (dataset is on uniform grid) - src_front_end: ', src_front_end print 'using time interpolation for co-location' if src_front_end == None: num_time_steps = num_time_steps0 num_lat = num_lat0 num_lon = num_lon0 else: (num_time_steps, num_lat, num_lon) = src_front_end.get_src_uniform_grid_info() # save the values for reuse possibly in the next i loop num_time_steps0 = num_time_steps num_lat0 = num_lat num_lon0 = num_lon midEnd.set_src_uniform_grid_info(num_time_steps, num_lat, num_lon) start1 = datetime.datetime.now() midEnd.co_locate(afl, afl1) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print '*** midEnd.co_locate elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print '****** midEnd.good_cnt: ', midEnd.good_cnt print '****** success match rate (%): ', ( 100.0 * midEnd.good_cnt) / midEnd.get_target_time( ).size, 'for target granule ', afl1.granuleNumList[i] if midEnd.good_cnt == 0: print 'No co-located data. Will not call the back-end. Skip to the next target granule.' continue start1 = datetime.datetime.now() # set target file name tgt_file_name = os.path.basename(tgt_file) dp = tgt_file_name.find('.') tf_name = tgt_file_name[0:dp] # for visualization of nearest neighbors and footprints # this is not essential to the product and can be commented out ### midEnd.set_output_filename(afl.outputdir+'/'+'latlon_'+src_data+'_'+tf_name+'.txt') ### midEnd.output_latlon() # pass data to back end bakEnd.set_target_time(midEnd.get_target_time()) bakEnd.set_target_lat(midEnd.get_target_lat()) bakEnd.set_target_lon(midEnd.get_target_lon()) bakEnd.set_target_data(midEnd.get_target_data()) bakEnd.set_invalid_data(midEnd.get_invalid_data()) ### print 'before set_target_levels(), src_front_end: ', src_front_end bakEnd.set_target_levels(src_front_end.get_levels()) # write out to output file bakEnd.set_output_filename(afl.outputdir + '/' + src_data + '_' + tf_name + '.nc') bakEnd.write_output(afl) now1 = datetime.datetime.now() elapsed_time = now1 - start1 print '*** backend elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' # To aid when remote profiling only #time.sleep(60) pre_idx1 = idx1 pre_idx2 = idx2 # end of for i loop (over target granules) now = datetime.datetime.now() elapsed_time = now - start print '*** sdriver elapsed time: ', elapsed_time.days, ' days, ', elapsed_time.seconds, ' seconds' print '--------- exiting sdriver(), gid: ', gid, ', pid: ', pid, ' ----------'