def convert(self, config_file_path):
with open(config_file_path) as config_file:
config_object = yaml.load(config_file)
ctl_file_path = os.path.abspath(config_object['ctl'])
grads_ctl = GradsCtl()
# output parser
output_dir = os.path.abspath(config_object['output_dir'])
# time parser
start_time_str = config_object.get('start_time', '')
forecast_time_str = config_object.get('forecast_time', '')
if start_time_str != "":
str_length = len(start_time_str)
if str_length == 10:
start_time = datetime.datetime.strptime(
start_time_str, "%Y%m%d%H")
grads_ctl.start_time = start_time
else:
logger.error(
"parser start_time has error: {start_time}".format(
start_time=start_time_str))
if forecast_time_str != "":
# TODO (windroc, 2014.08.18): use format:
# XXXhXXmXXs
if len(forecast_time_str) == 3:
forecast_time = datetime.timedelta(
hours=int(forecast_time_str))
grads_ctl.forecast_time = forecast_time
# ctl parser
grads_ctl_parser = GradsCtlParser(grads_ctl)
grads_ctl = grads_ctl_parser.parse(ctl_file_path)
# record parser
records = config_object['records']
for a_record in records:
target_type = a_record["target_type"]
self.print_record_info(a_record)
def convert_a_record():
if target_type.startswith("micaps"):
grads_to_micaps = GradsToMicaps(grads_ctl)
a_record['output_dir'] = output_dir
grads_to_micaps.convert(a_record)
else:
raise NotImplemented("Not implemented for %s" %
target_type)
time1 = time.clock()
convert_a_record()
time2 = time.clock()
logger.info("{time_cost:.2f}".format(time_cost=time2 - time1))
def test_get_offset_by_record_index(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.get_offset_by_record_index(0) == 0
assert data_handler.get_offset_by_record_index(3) == (751 * 501 + 2) * 4 * 3
with pytest.raises(ValueError):
data_handler.get_offset_by_record_index(6)
def test_get_record_list(self):
grads_ctl_parser = GradsCtlParser()
ctl_file_path = Path(
Path(__file__).parent.parent,
"data/ctl/meso/post.ctl_201802120000000")
grads_ctl = grads_ctl_parser.parse(str(ctl_file_path))
grads_copy = GradsCopy(where='var=t')
record_list = grads_copy.get_filtered_record_list(grads_ctl)
assert len(record_list) == 26
for a_record in record_list:
assert a_record['name'] == 't'
def test_get_offset_by_record_index(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.get_offset_by_record_index(0) == 0
assert data_handler.get_offset_by_record_index(3) == (751 * 501 +
2) * 4 * 3
with pytest.raises(ValueError):
data_handler.get_offset_by_record_index(6)
def test_get_record_by_index(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.get_record_by_index(0, 0).record_index == 0
assert data_handler.get_record_by_index(1, 1).record_index == 4
with pytest.raises(ValueError):
data_handler.get_record_by_index(0, 3)
with pytest.raises(ValueError):
data_handler.get_record_by_index(2, 0)
def test_get_record_by_index(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.get_record_by_index(0, 0).record_index == 0
assert data_handler.get_record_by_index(1, 1).record_index == 4
with pytest.raises(ValueError):
data_handler.get_record_by_index(0, 3)
with pytest.raises(ValueError):
data_handler.get_record_by_index(2, 0)
def test_find_record(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.find_record('t', 1000.0).record_index == 0
assert data_handler.find_record('t', 975.0).record_index == 1
assert data_handler.find_record('t', 950.0).record_index == 2
assert data_handler.find_record('h', 1000.0).record_index == 3
assert data_handler.find_record('h', 975.0).record_index == 4
assert data_handler.find_record('h', 950.0).record_index == 5
assert data_handler.find_record('t', 925.0) is None
assert data_handler.find_record('u', 1000.0) is None
def test_find_record(self):
ctl_parser = GradsCtlParser()
ctl_parser.parse(str(self.test_ctl_path))
grads_ctl = ctl_parser.grads_ctl
data_handler = GradsDataHandler(grads_ctl)
assert data_handler.find_record('t', 1000.0).record_index == 0
assert data_handler.find_record('t', 975.0).record_index == 1
assert data_handler.find_record('t', 950.0).record_index == 2
assert data_handler.find_record('h', 1000.0).record_index == 3
assert data_handler.find_record('h', 975.0).record_index == 4
assert data_handler.find_record('h', 950.0).record_index == 5
assert data_handler.find_record('t', 925.0) is None
assert data_handler.find_record('u', 1000.0) is None
def __init__(self, grads_ctl=None):
if grads_ctl is None:
grads_ctl = GradsCtl()
self.grads_ctl = grads_ctl
self.grads_ctl_parser = GradsCtlParser(grads_ctl)
self.grads_data_parser = GradsDataHandler(grads_ctl)
class GradsToMicaps(object):
"""
Convert GrADS data to micaps data
"""
def __init__(self, grads_ctl=None):
if grads_ctl is None:
grads_ctl = GradsCtl()
self.grads_ctl = grads_ctl
self.grads_ctl_parser = GradsCtlParser(grads_ctl)
self.grads_data_parser = GradsDataHandler(grads_ctl)
def set_grads_ctl_path(self, ctl_path):
self.grads_ctl_parser.parse(ctl_path)
def convert(self, a_config_record):
a_name = a_config_record['name']
a_level = a_config_record.get('level', '0')
a_level_type = a_config_record.get('level_type', 'multi')
an_output_dir = a_config_record.get('output_dir', '.')
a_time_index = a_config_record.get('time_index', 0)
value_func = eval("lambda x: {expression}".format(expression=a_config_record.get('value', 'x')))
record_target_type = a_config_record.get('target_type', '')
if record_target_type == "micaps.4":
self.convert_record_to_type_4(
a_name,
a_level,
a_level_type,
a_time_index,
an_output_dir,
value_func)
else:
print("TYPE: {record_target_type} has not implemented!".format(record_target_type=record_target_type))
def convert_record_to_type_4(
self, name,
level=0.0,
level_type='multi',
time_index=0,
output_dir=".",
value_func=lambda x: x):
"""
convert a record with name, level and time index in GrADS data file.
"""
micaps_data_type = "4"
a_forecast_hour = self.grads_ctl.forecast_time.seconds / 3600
comment = name + '_'+self.grads_ctl.start_time.strftime("%Y%m%d%H") + "_%03d" % a_forecast_hour
output_file_name = self.grads_ctl.start_time.strftime("%Y%m%d%H") + ".%03d" % a_forecast_hour
output_file_dir = output_dir + os.sep + name + "_" + micaps_data_type
if not level_type == 'single':
output_file_dir += os.sep + str(int(level))
a_level = float(level)
else:
a_level = 0
record = self.grads_data_parser.find_record(name, level, level_type)
with open(self.grads_ctl.dset, 'rb') as data_file:
record.load_data(data_file)
var_list = record.data.ravel()
vfunc = np.vectorize(value_func)
var_list = vfunc(var_list)
from porter.micaps_tool.writer.type_4 import MicapsType4Data, MicapsType4Writer
micaps_data = MicapsType4Data()
micaps_data.comment = comment
micaps_data.start_time = self.grads_ctl.start_time
micaps_data.forecast_hour = a_forecast_hour
micaps_data.level = a_level
micaps_data.x_step = self.grads_ctl.xdef['step']
micaps_data.y_step = self.grads_ctl.ydef['step']
micaps_data.x_start_value = self.grads_ctl.xdef['values'][0]
micaps_data.x_end_value = self.grads_ctl.xdef['values'][-1]
micaps_data.y_start_value = self.grads_ctl.ydef['values'][0]
micaps_data.y_end_value = self.grads_ctl.ydef['values'][-1]
micaps_data.x_count = self.grads_ctl.xdef['count']
micaps_data.y_count = self.grads_ctl.ydef['count']
micaps_data.contour_step = 4.00
micaps_data.contour_start_value = min(var_list)
micaps_data.contour_end_value = max(var_list)
micaps_data.smooth = 2
micaps_data.bold_value = 0.00
micaps_data.values = var_list
MicapsType4Writer.write_to_file(micaps_data, output_file_dir + os.sep + output_file_name)
if __name__ == "__main__":
import getopt
import sys
import struct
from porter.grads_parser.grads_ctl_parser import GradsCtlParser
optlist, args = getopt.getopt(sys.argv[1:], 'h')
if len(args) == 0:
print("""
Usage: %s ctl_file_path
""" % sys.argv[0])
sys.exit()
file_path = args[0]
ctl_parser = GradsCtlParser()
ctl_parser.parse(file_path)
data_handler = GradsDataHandler()
grads_ctl = ctl_parser.grads_ctl
data_handler.grads_ctl = grads_ctl
# open data file
y_count = grads_ctl.ydef['count']
x_count = grads_ctl.xdef['count']
print("length of the record: %d " % (x_count * y_count * 4))
data_file = open(grads_ctl.dset, 'rb')
data_file.seek(data_handler.get_offset_by_index(2, 5))
record_length_str = data_file.read(4)
record_length = struct.unpack('>I', record_length_str)[0]
print("length written at the beginning of the record: %d " % record_length)
if __name__ == "__main__":
import getopt
import sys
import struct
from porter.grads_parser.grads_ctl_parser import GradsCtlParser
optlist, args = getopt.getopt(sys.argv[1:], 'h')
if len(args) == 0:
print("""
Usage: %s ctl_file_path
""" % sys.argv[0])
sys.exit()
file_path = args[0]
ctl_parser = GradsCtlParser()
ctl_parser.parse(file_path)
data_handler = GradsDataHandler()
grads_ctl = ctl_parser.grads_ctl
data_handler.grads_ctl = grads_ctl
# open data file
y_count = grads_ctl.ydef['count']
x_count = grads_ctl.xdef['count']
print("length of the record: %d " % (x_count * y_count * 4))
data_file = open(grads_ctl.dset, 'rb')
data_file.seek(data_handler.get_offset_by_index(2, 5))
record_length_str = data_file.read(4)
record_length = struct.unpack('>I', record_length_str)[0]
print("length written at the beginning of the record: %d " % record_length)
def __init__(self, grads_ctl=None):
if grads_ctl is None:
grads_ctl = GradsCtl()
self.grads_ctl = grads_ctl
self.grads_ctl_parser = GradsCtlParser(grads_ctl)
self.grads_data_parser = GradsDataHandler(grads_ctl)
class GradsToMicaps(object):
"""
Convert GrADS data to micaps data
"""
def __init__(self, grads_ctl=None):
if grads_ctl is None:
grads_ctl = GradsCtl()
self.grads_ctl = grads_ctl
self.grads_ctl_parser = GradsCtlParser(grads_ctl)
self.grads_data_parser = GradsDataHandler(grads_ctl)
def set_grads_ctl_path(self, ctl_path):
self.grads_ctl_parser.parse(ctl_path)
def convert(self, a_config_record):
a_name = a_config_record['name']
a_level = a_config_record.get('level', '0')
a_level_type = a_config_record.get('level_type', 'multi')
an_output_dir = a_config_record.get('output_dir', '.')
a_time_index = a_config_record.get('time_index', 0)
value_func = eval("lambda x: {expression}".format(expression=a_config_record.get('value', 'x')))
record_target_type = a_config_record.get('target_type', '')
if record_target_type == "micaps.4":
self.convert_record_to_type_4(
a_name,
a_level,
a_level_type,
a_time_index,
an_output_dir,
value_func)
else:
raise NotImplemented("TYPE: {record_target_type} has not implemented!".format(
record_target_type=record_target_type))
def convert_record_to_type_4(
self, name,
level=0.0,
level_type='multi',
time_index=0,
output_dir=".",
value_func=lambda x: x):
"""
convert a record with name, level and time index in GrADS data file.
"""
micaps_data_type = "4"
a_forecast_hour = self.grads_ctl.forecast_time.seconds / 3600
comment = name + '_'+self.grads_ctl.start_time.strftime("%Y%m%d%H") + "_%03d" % a_forecast_hour
output_file_name = self.grads_ctl.start_time.strftime("%Y%m%d%H") + ".%03d" % a_forecast_hour
output_file_dir = output_dir + os.sep + name + "_" + micaps_data_type
if not level_type == 'single':
output_file_dir += os.sep + str(int(level))
a_level = float(level)
else:
a_level = 0
record = self.grads_data_parser.find_record(name, level, level_type)
with open(self.grads_ctl.dset, 'rb') as data_file:
record.load_data(data_file)
var_list = record.data.ravel()
vfunc = np.vectorize(value_func)
var_list = vfunc(var_list)
from porter.micaps_tool.writer.type_4 import MicapsType4Data, MicapsType4Writer
micaps_data = MicapsType4Data()
micaps_data.comment = comment
micaps_data.start_time = self.grads_ctl.start_time
micaps_data.forecast_hour = a_forecast_hour
micaps_data.level = a_level
micaps_data.x_step = self.grads_ctl.xdef['step']
micaps_data.y_step = self.grads_ctl.ydef['step']
micaps_data.x_start_value = self.grads_ctl.xdef['values'][0]
micaps_data.x_end_value = self.grads_ctl.xdef['values'][-1]
micaps_data.y_start_value = self.grads_ctl.ydef['values'][0]
micaps_data.y_end_value = self.grads_ctl.ydef['values'][-1]
micaps_data.x_count = self.grads_ctl.xdef['count']
micaps_data.y_count = self.grads_ctl.ydef['count']
micaps_data.contour_step = 4.00
micaps_data.contour_start_value = min(var_list)
micaps_data.contour_end_value = max(var_list)
micaps_data.smooth = 2
micaps_data.bold_value = 0.00
micaps_data.values = var_list
MicapsType4Writer.write_to_file(micaps_data, output_file_dir + os.sep + output_file_name)
def process(self, ctl_file):
grads_ctl_parser = GradsCtlParser()
grads_ctl = grads_ctl_parser.parse(ctl_file)
filtered_record_list = self.get_filtered_record_list(grads_ctl)
self.generate_output(grads_ctl, filtered_record_list)
def process(self, ctl_file):
grads_ctl_parser = GradsCtlParser()
grads_ctl = grads_ctl_parser.parse(ctl_file)
filtered_record_list = self.get_filtered_record_list(grads_ctl)
self.generate_output(grads_ctl, filtered_record_list)