def convert(self, config_file_path): with open(config_file_path) as config_file: config_object = yaml.load(config_file) ctl_file_path = os.path.abspath(config_object['ctl']) grads_ctl = GradsCtl() # output parser output_dir = os.path.abspath(config_object['output_dir']) # time parser start_time_str = config_object.get('start_time', '') forecast_time_str = config_object.get('forecast_time', '') if start_time_str != "": str_length = len(start_time_str) if str_length == 10: start_time = datetime.datetime.strptime( start_time_str, "%Y%m%d%H") grads_ctl.start_time = start_time else: logger.error( "parser start_time has error: {start_time}".format( start_time=start_time_str)) if forecast_time_str != "": # TODO (windroc, 2014.08.18): use format: # XXXhXXmXXs if len(forecast_time_str) == 3: forecast_time = datetime.timedelta( hours=int(forecast_time_str)) grads_ctl.forecast_time = forecast_time # ctl parser grads_ctl_parser = GradsCtlParser(grads_ctl) grads_ctl = grads_ctl_parser.parse(ctl_file_path) # record parser records = config_object['records'] for a_record in records: target_type = a_record["target_type"] self.print_record_info(a_record) def convert_a_record(): if target_type.startswith("micaps"): grads_to_micaps = GradsToMicaps(grads_ctl) a_record['output_dir'] = output_dir grads_to_micaps.convert(a_record) else: raise NotImplemented("Not implemented for %s" % target_type) time1 = time.clock() convert_a_record() time2 = time.clock() logger.info("{time_cost:.2f}".format(time_cost=time2 - time1))
def test_get_offset_by_record_index(self): ctl_parser = GradsCtlParser() ctl_parser.parse(str(self.test_ctl_path)) grads_ctl = ctl_parser.grads_ctl data_handler = GradsDataHandler(grads_ctl) assert data_handler.get_offset_by_record_index(0) == 0 assert data_handler.get_offset_by_record_index(3) == (751 * 501 + 2) * 4 * 3 with pytest.raises(ValueError): data_handler.get_offset_by_record_index(6)
def test_get_record_list(self): grads_ctl_parser = GradsCtlParser() ctl_file_path = Path( Path(__file__).parent.parent, "data/ctl/meso/post.ctl_201802120000000") grads_ctl = grads_ctl_parser.parse(str(ctl_file_path)) grads_copy = GradsCopy(where='var=t') record_list = grads_copy.get_filtered_record_list(grads_ctl) assert len(record_list) == 26 for a_record in record_list: assert a_record['name'] == 't'
def test_get_record_by_index(self): ctl_parser = GradsCtlParser() ctl_parser.parse(str(self.test_ctl_path)) grads_ctl = ctl_parser.grads_ctl data_handler = GradsDataHandler(grads_ctl) assert data_handler.get_record_by_index(0, 0).record_index == 0 assert data_handler.get_record_by_index(1, 1).record_index == 4 with pytest.raises(ValueError): data_handler.get_record_by_index(0, 3) with pytest.raises(ValueError): data_handler.get_record_by_index(2, 0)
def test_find_record(self): ctl_parser = GradsCtlParser() ctl_parser.parse(str(self.test_ctl_path)) grads_ctl = ctl_parser.grads_ctl data_handler = GradsDataHandler(grads_ctl) assert data_handler.find_record('t', 1000.0).record_index == 0 assert data_handler.find_record('t', 975.0).record_index == 1 assert data_handler.find_record('t', 950.0).record_index == 2 assert data_handler.find_record('h', 1000.0).record_index == 3 assert data_handler.find_record('h', 975.0).record_index == 4 assert data_handler.find_record('h', 950.0).record_index == 5 assert data_handler.find_record('t', 925.0) is None assert data_handler.find_record('u', 1000.0) is None
def __init__(self, grads_ctl=None): if grads_ctl is None: grads_ctl = GradsCtl() self.grads_ctl = grads_ctl self.grads_ctl_parser = GradsCtlParser(grads_ctl) self.grads_data_parser = GradsDataHandler(grads_ctl)
class GradsToMicaps(object): """ Convert GrADS data to micaps data """ def __init__(self, grads_ctl=None): if grads_ctl is None: grads_ctl = GradsCtl() self.grads_ctl = grads_ctl self.grads_ctl_parser = GradsCtlParser(grads_ctl) self.grads_data_parser = GradsDataHandler(grads_ctl) def set_grads_ctl_path(self, ctl_path): self.grads_ctl_parser.parse(ctl_path) def convert(self, a_config_record): a_name = a_config_record['name'] a_level = a_config_record.get('level', '0') a_level_type = a_config_record.get('level_type', 'multi') an_output_dir = a_config_record.get('output_dir', '.') a_time_index = a_config_record.get('time_index', 0) value_func = eval("lambda x: {expression}".format(expression=a_config_record.get('value', 'x'))) record_target_type = a_config_record.get('target_type', '') if record_target_type == "micaps.4": self.convert_record_to_type_4( a_name, a_level, a_level_type, a_time_index, an_output_dir, value_func) else: print("TYPE: {record_target_type} has not implemented!".format(record_target_type=record_target_type)) def convert_record_to_type_4( self, name, level=0.0, level_type='multi', time_index=0, output_dir=".", value_func=lambda x: x): """ convert a record with name, level and time index in GrADS data file. """ micaps_data_type = "4" a_forecast_hour = self.grads_ctl.forecast_time.seconds / 3600 comment = name + '_'+self.grads_ctl.start_time.strftime("%Y%m%d%H") + "_%03d" % a_forecast_hour output_file_name = self.grads_ctl.start_time.strftime("%Y%m%d%H") + ".%03d" % a_forecast_hour output_file_dir = output_dir + os.sep + name + "_" + micaps_data_type if not level_type == 'single': output_file_dir += os.sep + str(int(level)) a_level = float(level) else: a_level = 0 record = self.grads_data_parser.find_record(name, level, level_type) with open(self.grads_ctl.dset, 'rb') as data_file: record.load_data(data_file) var_list = record.data.ravel() vfunc = np.vectorize(value_func) var_list = vfunc(var_list) from porter.micaps_tool.writer.type_4 import MicapsType4Data, MicapsType4Writer micaps_data = MicapsType4Data() micaps_data.comment = comment micaps_data.start_time = self.grads_ctl.start_time micaps_data.forecast_hour = a_forecast_hour micaps_data.level = a_level micaps_data.x_step = self.grads_ctl.xdef['step'] micaps_data.y_step = self.grads_ctl.ydef['step'] micaps_data.x_start_value = self.grads_ctl.xdef['values'][0] micaps_data.x_end_value = self.grads_ctl.xdef['values'][-1] micaps_data.y_start_value = self.grads_ctl.ydef['values'][0] micaps_data.y_end_value = self.grads_ctl.ydef['values'][-1] micaps_data.x_count = self.grads_ctl.xdef['count'] micaps_data.y_count = self.grads_ctl.ydef['count'] micaps_data.contour_step = 4.00 micaps_data.contour_start_value = min(var_list) micaps_data.contour_end_value = max(var_list) micaps_data.smooth = 2 micaps_data.bold_value = 0.00 micaps_data.values = var_list MicapsType4Writer.write_to_file(micaps_data, output_file_dir + os.sep + output_file_name)
if __name__ == "__main__": import getopt import sys import struct from porter.grads_parser.grads_ctl_parser import GradsCtlParser optlist, args = getopt.getopt(sys.argv[1:], 'h') if len(args) == 0: print(""" Usage: %s ctl_file_path """ % sys.argv[0]) sys.exit() file_path = args[0] ctl_parser = GradsCtlParser() ctl_parser.parse(file_path) data_handler = GradsDataHandler() grads_ctl = ctl_parser.grads_ctl data_handler.grads_ctl = grads_ctl # open data file y_count = grads_ctl.ydef['count'] x_count = grads_ctl.xdef['count'] print("length of the record: %d " % (x_count * y_count * 4)) data_file = open(grads_ctl.dset, 'rb') data_file.seek(data_handler.get_offset_by_index(2, 5)) record_length_str = data_file.read(4) record_length = struct.unpack('>I', record_length_str)[0] print("length written at the beginning of the record: %d " % record_length)
class GradsToMicaps(object): """ Convert GrADS data to micaps data """ def __init__(self, grads_ctl=None): if grads_ctl is None: grads_ctl = GradsCtl() self.grads_ctl = grads_ctl self.grads_ctl_parser = GradsCtlParser(grads_ctl) self.grads_data_parser = GradsDataHandler(grads_ctl) def set_grads_ctl_path(self, ctl_path): self.grads_ctl_parser.parse(ctl_path) def convert(self, a_config_record): a_name = a_config_record['name'] a_level = a_config_record.get('level', '0') a_level_type = a_config_record.get('level_type', 'multi') an_output_dir = a_config_record.get('output_dir', '.') a_time_index = a_config_record.get('time_index', 0) value_func = eval("lambda x: {expression}".format(expression=a_config_record.get('value', 'x'))) record_target_type = a_config_record.get('target_type', '') if record_target_type == "micaps.4": self.convert_record_to_type_4( a_name, a_level, a_level_type, a_time_index, an_output_dir, value_func) else: raise NotImplemented("TYPE: {record_target_type} has not implemented!".format( record_target_type=record_target_type)) def convert_record_to_type_4( self, name, level=0.0, level_type='multi', time_index=0, output_dir=".", value_func=lambda x: x): """ convert a record with name, level and time index in GrADS data file. """ micaps_data_type = "4" a_forecast_hour = self.grads_ctl.forecast_time.seconds / 3600 comment = name + '_'+self.grads_ctl.start_time.strftime("%Y%m%d%H") + "_%03d" % a_forecast_hour output_file_name = self.grads_ctl.start_time.strftime("%Y%m%d%H") + ".%03d" % a_forecast_hour output_file_dir = output_dir + os.sep + name + "_" + micaps_data_type if not level_type == 'single': output_file_dir += os.sep + str(int(level)) a_level = float(level) else: a_level = 0 record = self.grads_data_parser.find_record(name, level, level_type) with open(self.grads_ctl.dset, 'rb') as data_file: record.load_data(data_file) var_list = record.data.ravel() vfunc = np.vectorize(value_func) var_list = vfunc(var_list) from porter.micaps_tool.writer.type_4 import MicapsType4Data, MicapsType4Writer micaps_data = MicapsType4Data() micaps_data.comment = comment micaps_data.start_time = self.grads_ctl.start_time micaps_data.forecast_hour = a_forecast_hour micaps_data.level = a_level micaps_data.x_step = self.grads_ctl.xdef['step'] micaps_data.y_step = self.grads_ctl.ydef['step'] micaps_data.x_start_value = self.grads_ctl.xdef['values'][0] micaps_data.x_end_value = self.grads_ctl.xdef['values'][-1] micaps_data.y_start_value = self.grads_ctl.ydef['values'][0] micaps_data.y_end_value = self.grads_ctl.ydef['values'][-1] micaps_data.x_count = self.grads_ctl.xdef['count'] micaps_data.y_count = self.grads_ctl.ydef['count'] micaps_data.contour_step = 4.00 micaps_data.contour_start_value = min(var_list) micaps_data.contour_end_value = max(var_list) micaps_data.smooth = 2 micaps_data.bold_value = 0.00 micaps_data.values = var_list MicapsType4Writer.write_to_file(micaps_data, output_file_dir + os.sep + output_file_name)
def process(self, ctl_file): grads_ctl_parser = GradsCtlParser() grads_ctl = grads_ctl_parser.parse(ctl_file) filtered_record_list = self.get_filtered_record_list(grads_ctl) self.generate_output(grads_ctl, filtered_record_list)