def radar_map_at_time(rootdir, radar, time):
"""read a radar map of the 'radar' at the 'time' from the director 'rootdir'. 6分钟一个，但是不是都从00点开始，会有漏掉的数据，所以循环需要注意，目前采用按小时的方式
TODO: the radar time is UTC time, so we have to convert it to the Beijing time
Return
-------
the path of the map we need
"""
files_list = os.listdir(rootdir)
for i in range(0, len(files_list)):
print(files_list[i][-14:-6])
date_time_right = pytime.parse(time)
print(date_time_right)
print(files_list[i][-5:])
if files_list[i][-5:] == radar:
date_temp = pytime.parse(files_list[i][-14:-6])
if date_temp == date_time_right.date():
file_list = os.listdir(rootdir + '/' + files_list[i])
for j in range(len(file_list)):
print(file_list[j][-18:-4])
date_time_temp = project_util.parse_datetime(
file_list[j][-18:-4])
print(date_time_temp.hour)
if date_time_right.hour == date_time_temp.hour:
path = os.path.join(
rootdir + '/' + files_list[i] + '/', file_list[j])
print(path)
break
break
return path
def parse_datetime(str_datetime):
"""for example:'20150801000000' -> '2015-08-02 00:00:00' -> 2015-08-02 00:00:00"""
str_date = str_datetime[:8]
str_time = str_datetime[8:10] + ':' + str_datetime[
10:12] + ':' + str_datetime[12:14]
rain_date_time = pytime.parse(str_date + ' ' + str_time)
return rain_date_time
def date_convert(value):
# if isinstance(value, str):
try:
create_date = pytime.parse(value)
except Exception as e:
create_date = datetime.datetime.now().date()
# else:
# create_date = value.date()
return create_date
def str_to_utc(s: str, raise_exceptions: bool = True) -> Optional[datetime]:
tz_shift = None
try:
if '+' in s:
s, shift = s.split('+')
tz_shift = timedelta(hours=int(shift))
s = s.replace('T', ' ')
ret = pytime.parse(s)
return ret + tz_shift
except:
logging.exception('Error while parse datetime')
if raise_exceptions:
raise
else:
return None
def test_parse(self):
this1 = pytime.parse('2015-5-17') == datetime.date(2015, 5, 17)
self.assertTrue(this1)
this2 = pytime.parse('2015/5/17') == datetime.date(2015, 5, 17)
self.assertTrue(this2)
this3 = pytime.parse('2015-5-17 23:23:23') == datetime.datetime(2015, 5, 17, 23, 23, 23)
self.assertTrue(this3)
this4 = pytime.parse('15-5-17 23:23:23') == datetime.datetime(2015, 5, 17, 23, 23, 23)
self.assertTrue(this4)
this5 = pytime.parse('2015517') == datetime.date(2015, 5, 17)
self.assertTrue(this5)
this6 = pytime.parse('2015.5.17') == datetime.date(2015, 5, 17)
self.assertTrue(this6)
this7 = pytime.parse('15-5-17') == datetime.date(2015, 5, 17)
self.assertTrue(this7)
this8 = pytime.parse('15.5.17') == datetime.date(2015, 5, 17)
self.assertTrue(this8)
this9 = pytime.parse('15/5/17') == datetime.date(2015, 5, 17)
self.assertTrue(this9)
this10 = pytime.parse('5/17/2015') == datetime.date(2015, 5, 17)
self.assertTrue(this10)
this11 = pytime.parse('17/5/2015') == datetime.date(2015, 5, 17)
self.assertTrue(this11)
this12 = pytime.parse('15517 23:23:23') == datetime.datetime(2015, 5, 17, 23, 23, 23)
self.assertTrue(this12)
this13 = pytime.parse('2015517 23:23:23') == datetime.datetime(2015, 5, 17, 23, 23, 23)
self.assertTrue(this13)
this14 = pytime.parse(1420041600 + gmt8offset) == datetime.datetime(2015, 1, 1, 0, 0, 0)
self.assertTrue(this14)
def test_from_str(self):
self.assertRaises(exception.CanNotFormatError, pytime.parse, 'App.19st,2015')
# validating the use with blank spaces
self.assertEqual(datetime.date(2015, 1, 1), pytime.parse('Jan.1 st, 2015'))
self.assertEqual(datetime.date(2015, 1, 2), pytime.parse('January 2nd 2015'))
self.assertEqual(datetime.date(2015, 1, 3), pytime.parse('Jan, 3rd 2015'))
# validating the name of months and the returned datetime
self.assertEqual(datetime.date(2015, 1, 2), pytime.parse('Jan.2st,2015'))
self.assertEqual(datetime.date(2015, 2, 19), pytime.parse('Feb.19st,2015'))
self.assertEqual(datetime.date(2015, 3, 19), pytime.parse('Mar.19st,2015'))
self.assertEqual(datetime.date(2015, 4, 19), pytime.parse('Apr.19st,2015'))
self.assertEqual(datetime.date(2015, 5, 19), pytime.parse('May.19st,2015'))
self.assertEqual(datetime.date(2015, 6, 19), pytime.parse('Jun.19st,2015'))
self.assertEqual(datetime.date(2014, 7, 19), pytime.parse('Jul.19st,2014'))
self.assertEqual(datetime.date(2015, 8, 19), pytime.parse('Agu.19st,2015'))
self.assertEqual(datetime.date(2015, 9, 19), pytime.parse('Sep.19st,2015'))
self.assertEqual(datetime.date(2015, 10, 19), pytime.parse('Oct.19st,2015'))
self.assertEqual(datetime.date(2015, 11, 19), pytime.parse('Nov.19st,2015'))
self.assertEqual(datetime.date(2014, 12, 19), pytime.parse('Dec.19st,2014'))
def test_parse(self):
this1 = pytime.parse('2015-5-17') == datetime.date(2015, 5, 17)
self.assertTrue(this1)
this2 = pytime.parse('2015/5/17') == datetime.date(2015, 5, 17)
self.assertTrue(this2)
this3 = pytime.parse('2015-5-17 23:23:23') == datetime.datetime(
2015, 5, 17, 23, 23, 23)
self.assertTrue(this3)
this4 = pytime.parse('15-5-17 23:23:23') == datetime.datetime(
2015, 5, 17, 23, 23, 23)
self.assertTrue(this4)
this5 = pytime.parse('2015517') == datetime.date(2015, 5, 17)
self.assertTrue(this5)
this6 = pytime.parse('2015.5.17') == datetime.date(2015, 5, 17)
self.assertTrue(this6)
this7 = pytime.parse('15-5-17') == datetime.date(2015, 5, 17)
self.assertTrue(this7)
this8 = pytime.parse('15.5.17') == datetime.date(2015, 5, 17)
self.assertTrue(this8)
this9 = pytime.parse('15/5/17') == datetime.date(2015, 5, 17)
self.assertTrue(this9)
this10 = pytime.parse('5/17/2015') == datetime.date(2015, 5, 17)
self.assertTrue(this10)
this11 = pytime.parse('17/5/2015') == datetime.date(2015, 5, 17)
self.assertTrue(this11)
this12 = pytime.parse('15517 23:23:23') == datetime.datetime(
2015, 5, 17, 23, 23, 23)
self.assertTrue(this12)
this13 = pytime.parse('2015517 23:23:23') == datetime.datetime(
2015, 5, 17, 23, 23, 23)
self.assertTrue(this13)
this14 = pytime.parse(1420041600 + gmt8offset) == datetime.datetime(
2015, 1, 1, 0, 0, 0)
self.assertTrue(this14)
def test_from_str(self):
self.assertRaises(exception.CanNotFormatError, pytime.parse,
'App.19st,2015')
#validating the use with blank spaces
self.assertEqual(datetime.date(2015, 1, 1),
pytime.parse('Jan.1 st, 2015'))
self.assertEqual(datetime.date(2015, 1, 2),
pytime.parse('January 2nd 2015'))
self.assertEqual(datetime.date(2015, 1, 3),
pytime.parse('Jan, 3rd 2015'))
#validating the name of months and the returned datetime
self.assertEqual(datetime.date(2015, 1, 2),
pytime.parse('Jan.2st,2015'))
self.assertEqual(datetime.date(2015, 2, 19),
pytime.parse('Feb.19st,2015'))
self.assertEqual(datetime.date(2015, 3, 19),
pytime.parse('Mar.19st,2015'))
self.assertEqual(datetime.date(2015, 4, 19),
pytime.parse('Apr.19st,2015'))
self.assertEqual(datetime.date(2015, 5, 19),
pytime.parse('May.19st,2015'))
self.assertEqual(datetime.date(2015, 6, 19),
pytime.parse('Jun.19st,2015'))
self.assertEqual(datetime.date(2014, 7, 19),
pytime.parse('Jul.19st,2014'))
self.assertEqual(datetime.date(2015, 8, 19),
pytime.parse('Agu.19st,2015'))
self.assertEqual(datetime.date(2015, 9, 19),
pytime.parse('Sep.19st,2015'))
self.assertEqual(datetime.date(2015, 10, 19),
pytime.parse('Oct.19st,2015'))
self.assertEqual(datetime.date(2015, 11, 19),
pytime.parse('Nov.19st,2015'))
self.assertEqual(datetime.date(2014, 12, 19),
pytime.parse('Dec.19st,2014'))
def radar_rain_gauge_merge(start_time, end_time):
"""merge radar data with rain gauge"""
time_step_type = str(project_util.TimeUnitType.Hour.value)
time_step_length = 1
rain_gauge_sites_id = [
15, 19, 23, 82, 181, 182, 183, 251, 252, 254, 255, 256, 257, 258, 260,
261, 262, 263, 267, 268, 269, 270, 271, 272, 274, 322, 341, 361, 362
]
zs_rain_gauge = read_rain_gauge_data(rain_gauge_sites_id, start_time,
end_time, time_step_type,
time_step_length)
print(zs_rain_gauge.values)
print(zs_rain_gauge.values[0])
# read rain_gauge_site_num_in_radar_grid
radar_center_x = int(
project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_center_x'))
radar_center_y = int(
project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_center_y'))
radar_resolution_x = float(
project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_resolution_x'))
radar_resolution_y = float(
project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_resolution_y'))
x_in_radar_grid, y_in_radar_grid = rain_gauge_site_num_in_radar_grid(
rain_gauge_sites_id, radar_center_x, radar_center_y,
radar_resolution_x, radar_resolution_y)
periods_num = project_util.time_period_num(start_time, end_time,
time_step_type,
time_step_length)
datelist = pd.date_range(start_time, freq='H', periods=periods_num)
rootdir = project_util.read_radar_data_dir('config.ini', 'radar-data',
'data_directory')
radar = project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_code')
for i in range(len(datelist)):
radar_map_path = radar_map_at_time(rootdir, radar, datelist[i])
'''get volume-integrated grid data by radar. We have many radar graphs, but we can't read them one-time, because the
memory needed to load data is too big to be satisfied. For that reason, we need a loop and divide-and-conquer them
one by one '''
radar_data_interval = data_preprocess.read_precipitation_from_image(
radar_map_path)
'''krige radar data at the rain gauge locations'''
# read radar data at the rain gauge locations. Because the data of radar has the form of interval, we name it "range"
radar_data_in_rain_gauge_sites = read_radar_data_in_positions(
radar_data_interval, x_in_radar_grid, y_in_radar_grid)
# To krige, we have to convert the range to a number. Here, I choose the way to average the range
radar_radius_in_graph = int(
project_util.read_radar_data_dir('config.ini', 'radar-data',
'radar_radius_in_graph'))
z_radar, ss_radar = rain_ordinary_kriging(
radar_center_x, radar_center_y, radar_radius_in_graph,
radar_resolution_x, radar_resolution_y, x_in_radar_grid,
y_in_radar_grid, radar_data_in_rain_gauge_sites)
'''compute the deviation between the radar-observation and kriging-radar'''
# radar_data have to be sliced to adapt the size of kriging's range
radar_data = radar_data_interval.sum(2) / 2
x_start_index = radar_center_x - radar_radius_in_graph
y_start_index = radar_center_y - radar_radius_in_graph
x_end_index = int((radar_center_x - radar_radius_in_graph +
radar_radius_in_graph * 2) + 1)
y_end_index = int((radar_center_y - radar_radius_in_graph +
radar_radius_in_graph * 2) + 1)
deviation = radar_data[x_start_index:x_end_index,
y_start_index:y_end_index] - z_radar
'''apply deviation to kriging-rain-gauge'''
z_rain_gauge, ss_rain_gauge = rain_ordinary_kriging(
radar_center_x, radar_center_y, radar_radius_in_graph,
radar_resolution_x, radar_resolution_y, x_in_radar_grid,
y_in_radar_grid, zs_rain_gauge.values[i])
merge_data = deviation + z_rain_gauge
'''write data to database'''
rain_graph_time = radar_map_path[:-4][-14:]
rain_date = rain_graph_time[:8]
rain_time = rain_graph_time[8:10] + ':' + rain_graph_time[
10:12] + ':' + rain_graph_time[12:14]
rain_date_time = pytime.parse(rain_date + ' ' + rain_time)
# write_radar_merge_data(merge_data, rain_date_time)
return merge_data, x_start_index, y_start_index, x_end_index, y_end_index
from pytime import pytime
print(pytime.parse("20030925"))
column_lsit = [1, 2, 3, 4, 5, 6]
print(str(column_lsit)[1:-1])
df1 = pd.DataFrame(np.random.randn(len(datelist), len(column_lsit)),
index=datelist,
columns=column_lsit)
print(len(df1))
print(df1.iloc[:, [5, 4]])
#
# for row in a.iterrows():
# temp = row[1].values[0]
# print(temp)
# df1.loc[temp, 2] = 10000
# print(df1)
temp = pytime.parse("2018-05-05 00:00:00")
date_temp = pytime.parse("2018-05-05")
print(temp)
print(temp.date())
print(date_temp)
print(temp.date() == date_temp)
data = np.random.rand(4, 2, 2)
print(data)
print(data.sum(2))
# p = pd.Panel(data)
# print(p.values)
# print(p.iloc[[0, 1], [0, 1]].values)
# arr = xarray.DataArray(np.random.RandomState(0).randn(3, 3, 2),
# [(['a', 'b', 'c']), ([10, 20, 30]), ([1, 2])])
