def genetic_algorithm(args):
minmax = (args.min, args.max)
N = args.N
gens = args.gens
solution = common.initialize(N,minmax)
best_fitness = common.fitness(solution[0])
for gen in range(1, gens):
if gen % 10 == 0:
print("Generation :#%d" % gen)
mutated = mutation.ga_mutation(solution, minmax)
fitness = common.fitness(mutated[0])
if fitness >= best_fitness:
best_fitness = fitness
solution = mutated
common.write_data(gen, fitness, 'ga.dat')
print("#########################")
print("# Strategy : Genetic Algorithms")
print("# Generations : " + str(gens))
print("# Best Solution Fitness : %.3f" % best_fitness)
print("# Log File : ./ga.dat")
print("# Graph : Genetic_Algorithm_Ackleys_Function.png")
print("#########################")
common.plot('Genetic Algorithm: Ackleys Function', 'ga.dat')
def promice2nc(args, input_file, output_file, stations):
df = init_dataframe(args, input_file)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
common.log(args, 2, 'Retrieving latitude, longitude and station name')
latitude, longitude, station_name = get_station(args, input_file, stations)
common.log(args, 3, 'Calculating time and sza')
time, time_bounds, sza = get_time_and_sza(args, df, longitude, latitude)
common.log(args, 4, 'Converting lat_GPS and lon_GPS')
convert_coordinates(args, df)
common.log(args, 5, 'Calculating ice velocity')
fill_ice_velocity(args, df, ds)
ds['time'] = 'time', time
ds['time_bounds'] = ('time', 'nbnd'), time_bounds
ds['sza'] = 'time', sza
ds['station_name'] = tuple(), station_name
ds['latitude'] = tuple(), latitude
ds['longitude'] = tuple(), longitude
common.load_dataset_attributes('promice', ds)
encoding = common.get_encoding('promice', get_fillvalue(args))
common.write_data(args, ds, output_file, encoding)
def genetic_algorithm(args):
minmax = (args.min, args.max)
N = args.N
gens = args.gens
solution = common.initialize(N,minmax)
best_fitness = common.fitness(solution[0])
for gen in range(1, gens):
if gen % 10 == 0:
print("Generation :#%d" % gen)
mutated = mutation.ga_mutation(solution, minmax)
fitness = common.fitness(mutated[0])
if fitness <= best_fitness:
best_fitness = fitness
solution = mutated
common.write_data(gen, fitness, 'ga.dat')
if fitness == 0:
break
print("#########################")
print("# Strategy : Genetic Algorithms")
print("# Generations : " + str(gens))
print("# Best Solution Fitness : %.3f" % best_fitness)
print("# Log File : ./ga.dat")
print("# Graph : Genetic_Algorithm_Ackleys_Function.png")
print("#########################")
common.plot('Genetic Algorithm: Ackleys Function', 'ga.dat')
def clean(self, root_path, label_file, output_path):
times = 1
with open(label_file) as f:
for line in f.readlines():
line = line.replace('\r', '').replace('\n', '')
print(line)
list_str = line.split(' ')
image_path = os.path.join(root_path, list_str[0])
print(image_path)
img = cv2.imread(image_path)
h, w, c = img.shape
print(h, w, c)
if int(list_str[1]) != 4:
print('[ERROR] ' + list_str[0] + ' points not 4 !!')
cv2.circle(img, (int(float(list_str[2]) * times),
int(float(list_str[3])) * times), 3,
(0, 0, 255), -1)
cv2.circle(img, (int(float(list_str[4]) * times),
int(float(list_str[5])) * times), 3,
(0, 255, 255), -1)
cv2.circle(img, (int(float(list_str[6]) * times),
int(float(list_str[7])) * times), 3,
(255, 0, 0), -1)
cv2.circle(img, (int(float(list_str[8]) * times),
int(float(list_str[9])) * times), 3,
(0, 255, 0), -1)
cv2.imshow('check_image', img)
while True:
# 保存这张图片
k = cv2.waitKey(1) & 0xFF
if k == ord('y'):
save_path = os.path.join(output_path, list_str[0])
print(save_path)
(file_path, tempfilename) = os.path.split(save_path)
print(file_path)
mkdir_if_not_exist(file_path)
shutil.copy(image_path, save_path)
common.write_data(
os.path.join(output_path, 'label.txt'),
line + '\n', 'a+')
break
if k == ord('n'):
break
print(
"===================================================================="
)
return
def evolutionary_strategies(args):
minmax = (args.min, args.max)
N = args.N
gens = args.gens
solution = common.initialize(N, minmax)
best_fitness = common.fitness(solution[0])
p = 1.5
for gen in range(1, gens):
if gen % 10 == 0:
print("Generation :#%d" % gen)
mutated = mutation.es_mutation(solution, minmax, p)
#print(solution[0])
#print(mutated[0])
#print(list(map(operator.sub, mutated[0], solution[0])))
fitness = common.fitness(mutated[0])
if fitness <= best_fitness:
best_fitness = fitness
solution = mutated
p = 1.5
common.write_data(gen, fitness, 'es.dat')
#elif fitness == best_fitness:
# p = 1
else:
p = 1.5**(-1 / 4)
if fitness == 0:
break
print("#########################")
print("# Strategy : Evolutionary Strategies")
print("# Generations : " + str(gens))
print("# Best Solution Fitness : %.3f" % best_fitness)
print("# Log File : ./es.dat")
print(
"# Graph : Evolutionary_Strategies_Ackleys_Function.png"
)
print("#########################")
common.plot('Evolutionary Strategies: Ackleys Function', 'es.dat')
def clean_start(self, root_dir, label_path, restart=False):
with open(label_path, 'r') as f:
lines = f.readlines()
for line in lines:
str_list = line.split(' ')
image_path = os.path.join(root_dir, str_list[0])
img = cv2.imread(image_path)
while True:
cv2.imshow('clean_image', img)
k = cv2.waitKey(1) & 0xFF
if k == ord('y'):
print('save ...')
common.write_data(os.path.join('.', label_path.split('/')[-1]), line, 'a+')
break
if k == ord('n'):
break
return
def page_home():
file_name = "story_data.csv"
all_data = read_data(file_name)
current_data_id = request.args.get('delete_id')
del_data = []
# remove data
for data in all_data:
if data[0] == current_data_id:
del_data = data
if del_data != []:
all_data.remove(del_data)
write_data(file_name, all_data)
return redirect(url_for('page_home'))
return render_template("list.html", user_stories=all_data)
def evolutionary_strategies(args):
minmax = (args.min, args.max)
N = args.N
gens = args.gens
solution = common.initialize(N,minmax)
best_fitness = common.fitness(solution[0])
p = 1.5
for gen in range(1, gens):
if gen % 10 == 0:
print("Generation :#%d" % gen)
mutated = mutation.es_mutation(solution, minmax, p)
#print(solution[0])
#print(mutated[0])
#print(list(map(operator.sub, mutated[0], solution[0])))
fitness = common.fitness(mutated[0])
if fitness <= best_fitness:
best_fitness = fitness
solution = mutated
p = 1.5
common.write_data(gen, fitness, 'es.dat')
#elif fitness == best_fitness:
# p = 1
else:
p = 1.5 ** (-1/4)
if fitness == 0:
break
print("#########################")
print("# Strategy : Evolutionary Strategies")
print("# Generations : " + str(gens))
print("# Best Solution Fitness : %.3f" % best_fitness)
print("# Log File : ./es.dat")
print("# Graph : Evolutionary_Strategies_Ackleys_Function.png")
print("#########################")
common.plot('Evolutionary Strategies: Ackleys Function', 'es.dat')
def page_update_story(story_id):
title = "Super Sprinter 3000 - Edit Story"
button_name = "update"
file_name = "story_data.csv"
all_data = read_data(file_name)
current_data_id = story_id
# get the current updating data index
for data in all_data:
if data[0] == current_data_id:
current_data = data
current_data_index = all_data.index(current_data)
if request.method == "POST":
update_data = [
current_data_id,
request.form['story_title'],
request.form['user_story'],
request.form['acceptance_criteria'],
request.form['business_value'],
request.form['estimation'],
request.form['status']
]
all_data[current_data_index] = update_data
write_data(file_name, all_data)
return redirect(url_for('page_home'))
return render_template(
"form.html",
story_title=all_data[current_data_index][1],
user_story=all_data[current_data_index][2],
acceptance_criteria=all_data[current_data_index][3],
business_value=all_data[current_data_index][4],
estimation=all_data[current_data_index][5],
status=all_data[current_data_index][6],
title=title,
button_name=button_name
)
def page_add_story():
title = "Super Sprinter 3000 - Add new Story"
button_name = "create"
file_name = "story_data.csv"
all_data = read_data(file_name)
data = []
if request.method == "POST":
data.append(str(generate_data_id(file_name)))
data.append(request.form['story_title'])
data.append(request.form['user_story'])
data.append(request.form['acceptance_criteria'])
data.append(request.form['business_value'])
data.append(request.form['estimation'])
data.append(request.form['status'])
all_data.append(data)
write_data(file_name, all_data)
return redirect(url_for('page_home'))
return render_template("form.html", title=title, button_name=button_name)
def aaws2nc(args, input_file, output_file, stations):
df = init_dataframe(args, input_file)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
common.log(args, 2, 'Retrieving latitude, longitude and station name')
latitude, longitude, station_name = get_station(args, input_file, stations)
common.log(args, 3, 'Calculating time and sza')
time, time_bounds, sza = get_time_and_sza(
args, input_file, latitude, longitude)
ds['time'] = 'time', time
ds['time_bounds'] = ('time', 'nbnd'), time_bounds
ds['sza'] = 'time', sza
ds['station_name'] = tuple(), station_name
ds['latitude'] = tuple(), latitude
ds['longitude'] = tuple(), longitude
common.load_dataset_attributes('aaws', ds)
encoding = common.get_encoding('aaws', get_fillvalue(args))
common.write_data(args, ds, output_file, encoding)
def gcnet2nc(args, input_file, output_file, stations):
df = init_dataframe(args, input_file)
station_number = df['station_number'][0]
df.drop('station_number', axis=1, inplace=True)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
common.log(args, 2, 'Retrieving latitude, longitude and station name')
latitude, longitude, station_name = get_station(args, input_file, stations)
common.log(args, 3, 'Calculating time and sza')
hour, month, day, time, time_bounds, sza = get_time_and_sza(
args, df, longitude, latitude)
common.log(args, 4, 'Calculating quality control variables')
fill_dataset_quality_control(df, ds)
if args.derive_times:
common.log(args, 5, 'Calculating month and day')
derive_times(df, month, day)
ds['hour'] = 'time', hour
ds['month'] = 'time', month
ds['day'] = 'time', day
ds['time'] = 'time', time
ds['time_bounds'] = ('time', 'nbnd'), time_bounds
ds['sza'] = 'time', sza
ds['station_number'] = tuple(), station_number
ds['station_name'] = tuple(), station_name
ds['latitude'] = tuple(), latitude
ds['longitude'] = tuple(), longitude
common.load_dataset_attributes('gcnet', ds)
encoding = common.get_encoding('gcnet', get_fillvalue(args))
common.write_data(args, ds, output_file, encoding)
def page_questions():
file_name = "question.csv"
title = "ASK A QUESTION"
button_name = "Post your question"
all_data = read_data(file_name)
timestamp = int(time.time())
data_list = []
if request.method == "POST":
data_list.append(str(generate_data_id(file_name)))
data_list.append(str(timestamp))
data_list.append(' ') # view number
data_list.append(' ') # vote number
data_list.append(request.form['question_title'])
data_list.append(request.form['message'])
data_list.append(' ') # for picture
all_data.append(data_list)
new_data_to_write = write_data(file_name, all_data)
return redirect(url_for('page_home'))
return render_template("get_data.html",
title=title,
button_name=button_name)
def new_answer(question_id):
question_database = read_data('question.csv')
for line in question_database:
if str(question_id) in line[0]:
question_line = line
file_name = "answer.csv"
button_name = "Post your answer"
all_data = read_data(file_name)
timestamp = int(time.time())
data_list = []
if request.method == "POST":
data_list.append(str(generate_data_id(file_name)))
data_list.append(str(timestamp))
data_list.append(' ') # view number
data_list.append(question_id)
data_list.append(request.form['message'])
data_list.append(' ') # for picture
all_data.append(data_list)
new_data_to_write = write_data(file_name, all_data)
return redirect(url_for('all_answers', question_id=question_id))
return render_template("add_answer.html", question_line=question_line)
while True:
cv2.imshow('sign_image', self.img)
# 保存这张图片
k = cv2.waitKey(1) & 0xFF
if k == ord('s') or k == ord('1'):
print('save ...')
data = self.img_files[start_i] + " " + str(
len(self.car_points))
for (x, y) in self.car_points:
data += ' ' + str(x / float(times)) + ' ' + str(
y / float(times))
data += '\n'
common.write_data(self.label_file, data, 'a+')
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
self.car_points = []
break
if k == ord('2'):
print('save and copy ...')
data = self.img_files[start_i] + " " + str(
len(self.car_points))
for (x, y) in self.car_points:
data += ' ' + str(x / float(times)) + ' ' + str(
y / float(times))
data += '\n'
common.write_data(self.label_file, data, 'a+')
# print(self.img_files[start_i])
# print(type(self.img_files[start_i]))
self.img = cv2.imread(self.img_files[start_i])
self.img = cv2.resize(self.img, (self.img.shape[1]*times, self.img.shape[0]*times))
cv2.imshow('sign_image', self.img)
while True:
cv2.imshow('sign_image', self.img)
# 保存这张图片
k = cv2.waitKey(1) & 0xFF
if k == ord('s'):
self.save_label(self.img_files[start_i], plate)
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
break
if k == ord('1'):
self.save_label(self.img_files[start_i], plate, 1)
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
break
if k == ord('2'):
self.save_label(self.img_files[start_i], plate, 2)
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
break
if k == ord('3'):
found_imgs = [img_path for img_path in self.img_files_2 if img_id + "_" in img_path]
print(found_imgs)
for i, img_path in enumerate(found_imgs):
img_2 = cv2.imread(img_path)
img_2 = cv2.resize(img_2, (img_2.shape[1]*times, img_2.shape[0]*times))
cv2.imshow('image_' + str(i), img_2)
while True:
k = cv2.waitKey(1) & 0xFF
if k == ord('y'):
print('copy image ...')
# shutil.move(self.img_files_2[start_i], self.output_dir)
shutil.copy(old_img, self.output_dir)
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
break
if k == ord('n'):
start_i += 1
common.write_data(self.index_file, str(start_i), 'w')
break
# cv2.destroyAllWindows()
if __name__ == '__main__':
# image_dir_1 = "../capture_image/province_nosign/failed_recognize/"
# image_dir_2 = "../capture_image/province_nosign/failed_recognize_routh/"
# image_dir_1 = "../capture_image/province_nosign/failed_detect/"
# image_dir_2 = "../capture_image/province_nosign/failed_detect_routh/"
def save_label(self, file_name, plate, times=1):
print('save_label ...')
data = file_name + ":" + plate.encode(self.plate_encode) + '\n'
print('[save_label] plate len: %d' % len(plate))
if len(plate) == 7: # 正常车牌
if times == 1:
common.write_data(self.label_normal_file, data, 'a+')
else:
for i in range(times):
common.write_data(self.label_normal_file, data, 'a+')
common.write_data(self.label_test_file, data, 'a+')
elif len(plate) == 8: # 新能源车牌
if times == 1:
common.write_data(self.label_normal_file, data, 'a+')
else:
for i in range(times):
common.write_data(self.label_normal_file, data, 'a+')
common.write_data(self.label_test_file, data, 'a+')
else: # 其他车牌
common.write_data(self.label_error_file, data, 'a+')
parser.add_argument("--max", help="Maximum value", type=int)
parser.add_argument('-g', "--generations", help="Number of Generations",
dest='gens', type=int)
args = parser.parse_args()
minmax = (args.min, args.max)
N = args.N
gens = args.gens
solution = common.initialize(N,minmax)
best_fitness = common.fitness(solution[0])
p = 1.5
for gen in range(1, gens):
mutated = mutation.es_mutation(solution, minmax, p)
fitness = common.fitness(mutated[0])
if fitness <= best_fitness:
best_fitness = fitness
solution = mutated
p = 1.5
common.write_data(gen, fitness, 'data.dat')
else:
p = 1.5 ** (-1/4)
if fitness == 0:
break
def evolutionary_strategies(args):
minmax = common.get_minmax(args.fitness)
N = args.N
gens = args.gens
population_exchange = args.exchange
exchange_individuals = args.iexchange
islands = args.islands
population_list = []
fittest_list = []
best_fitness_list = []
for i in range(0,islands):
population = common.initialize(N,minmax)
fittest, best_fitness = common.fittest(population[0],
args.fitness)
population_list.append(population)
fittest_list.append(fittest)
best_fitness_list.append(best_fitness)
p_list = [0] * islands
successful_cases_list = [0] * islands
for gen in range(1, gens):
if gen % (gens / 10) == 0:
print("Generation :#%d" % gen)
if gen % population_exchange == 0:
print(" -> Exchange in Generation: %d " % gen)
population_list = common.exchange(population_list,
exchange_individuals)
mutated = [[]] * islands
local_fittest = [0] * islands
fitness = [0] * islands
for i in range(0, islands):
mutated[i] = mutation.es_mutation(population_list[i],
minmax, p_list[i])
local_fittest[i], fitness[i] = common.fittest(mutated[i][0],
args.fitness)
if fitness[i] >= best_fitness_list[i]:
fittest_list[i] = local_fittest[i]
best_fitness_list[i] = fitness[i]
successful_cases_list[i] += 1
population = mutated
common.write_data(gen, fitness[i], 'es%d.dat' % i)
p_list[i] = successful_cases_list[i] / gen
print("#########################")
print("# Strategy : Evolutionary Strategies")
print("# Generations : " + str(gens))
fittest = 0
best_fitness = 0
best_island = 0
for i in range(0, islands):
if best_fitness_list[i] > best_fitness:
fittest = fittest_list[i]
best_fitness = best_fitness_list[i]
best_island = i
print("# Best Solution Value : %.3f" % fittest)
print("# Best Solution Fitness : %g" % best_fitness)
print("# Obtained from Island : %d" % best_island)
print("# Log File : ./es%d.dat" % best_island)
print("# Graph : Evolutionary_Strategies_%s.png" %
args.fitness.upper())
print("#########################")
common.plot('Evolutionary Strategies %s' % args.fitness.upper(),
'es%d.dat' % best_island)
files = []
for island in range(0, islands):
common.plot('Evolutionary Strategies %s_islands' % args.fitness.upper(),
'es%d.dat' % island, False)
def gcnet2nc(args, op_file, station_dict, station_name, convert_temp, convert_press, seconds_in_hour, fillvalue_double):
header_rows = 54
check_na = 999.0
hour_conversion = (100/4) #Divided by 4 because each hour value is a multiple of 4 and then multiplied by 100 to convert decimal to integer
last_hour = 23
column_names = ['station_number', 'year', 'julian_decimal_time', 'sw_down', 'sw_up', 'net_radiation', 'temperature_tc_1', 'temperature_tc_2', 'temperature_cs500_1', 'temperature_cs500_2', 'relative_humidity_1', 'relative_humidity_2',
'u1_wind_speed', 'u2_wind_speed', 'u_direction_1', 'u_direction_2', 'atmos_pressure', 'snow_height_1', 'snow_height_2', 't_snow_01', 't_snow_02', 't_snow_03', 't_snow_04', 't_snow_05', 't_snow_06', 't_snow_07', 't_snow_08', 't_snow_09', 't_snow_10',
'battery_voltage', 'sw_down_max', 'sw_up_max', 'net_radiation_max', 'max_air_temperature_1', 'max_air_temperature_2', 'min_air_temperature_1', 'min_air_temperature_2', 'max_windspeed_u1', 'max_windspeed_u2', 'stdev_windspeed_u1', 'stdev_windspeed_u2',
'ref_temperature', 'windspeed_2m', 'windspeed_10m', 'wind_sensor_height_1', 'wind_sensor_height_2', 'albedo', 'zenith_angle', 'qc1', 'qc9', 'qc17', 'qc25']
df = pd.read_csv(args.input_file or args.fl_in, delim_whitespace=True, skiprows=header_rows, skip_blank_lines=True, header=None, names = column_names)
df.index.name = 'time'
df['qc25'] = df['qc25'].astype(str) # To avoid 999 values marked as N/A
df.replace(check_na, np.nan, inplace=True)
df.loc[:,['temperature_tc_1', 'temperature_tc_2', 'temperature_cs500_1', 'temperature_cs500_2', 't_snow_01', 't_snow_02', 't_snow_03', 't_snow_04', 't_snow_05', 't_snow_06', 't_snow_07', 't_snow_08', 't_snow_09', 't_snow_10', 'max_air_temperature_1', 'max_air_temperature_2', 'min_air_temperature_1', 'min_air_temperature_2', 'ref_temperature']] += convert_temp
df.loc[:,'atmos_pressure'] *= convert_press
df = df.where((pd.notnull(df)), fillvalue_double)
df['qc25'] = df['qc25'].astype(int) #Convert it back to int
station_number = df['station_number'][0]
df.drop('station_number', axis=1, inplace=True)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
# Intializing variables
num_rows = df['year'].size
qc_swdn, qc_swup, qc_netradiation, qc_ttc1, qc_ttc2, qc_tcs1, qc_tcs2, qc_rh1, qc_rh2, qc_u1, qc_u2, qc_ud1, qc_ud2, qc_pressure, qc_snowheight1, qc_snowheight2, qc_tsnow1, qc_tsnow2, qc_tsnow3, qc_tsnow4, qc_tsnow5, qc_tsnow6, qc_tsnow7, qc_tsnow8, qc_tsnow9, qc_tsnow10, qc_battery = ([0]*num_rows for x in range(27))
hour, month, day, time, time_bounds, sza = ([0]*num_rows for x in range(6))
print('calculating quality control variables...')
temp1 = [list(map(int, i)) for i in zip(*map(str, df['qc1']))]
temp9 = [list(map(int, i)) for i in zip(*map(str, df['qc9']))]
temp17 = [list(map(int, i)) for i in zip(*map(str, df['qc17']))]
temp25 = [list(map(int, i)) for i in zip(*map(str, df['qc25']))]
qc_swdn[:] = temp1[0]
qc_swup[:] = temp1[1]
qc_netradiation[:] = temp1[2]
qc_ttc1[:] = temp1[3]
qc_ttc2[:] = temp1[4]
qc_tcs1[:] = temp1[5]
qc_tcs2[:] = temp1[6]
qc_rh1[:] = temp1[7]
qc_rh2[:] = temp9[0]
qc_u1[:] = temp9[1]
qc_u2[:] = temp9[2]
qc_ud1[:] = temp9[3]
qc_ud2[:] = temp9[4]
qc_pressure[:] = temp9[5]
qc_snowheight1[:] = temp9[6]
qc_snowheight2[:] = temp9[7]
qc_tsnow1[:] = temp17[0]
qc_tsnow2[:] = temp17[1]
qc_tsnow3[:] = temp17[2]
qc_tsnow4[:] = temp17[3]
qc_tsnow5[:] = temp17[4]
qc_tsnow6[:] = temp17[5]
qc_tsnow7[:] = temp17[6]
qc_tsnow8[:] = temp17[7]
qc_tsnow9[:] = temp25[0]
qc_tsnow10[:] = temp25[1]
qc_battery[:] = temp25[2]
ds['qc_swdn'] = (('time',qc_swdn))
ds['qc_swup'] = (('time',qc_swup))
ds['qc_netradiation'] = (('time',qc_netradiation))
ds['qc_ttc1'] = (('time',qc_ttc1))
ds['qc_ttc2'] = (('time',qc_ttc2))
ds['qc_tcs1'] = (('time',qc_tcs1))
ds['qc_tcs2'] = (('time',qc_tcs2))
ds['qc_rh1'] = (('time',qc_rh1))
ds['qc_rh2'] = (('time',qc_rh2))
ds['qc_u1'] = (('time',qc_u1))
ds['qc_u2'] = (('time',qc_u2))
ds['qc_ud1'] = (('time',qc_ud1))
ds['qc_ud2'] = (('time',qc_ud2))
ds['qc_pressure'] = (('time',qc_pressure))
ds['qc_snowheight1'] = (('time',qc_snowheight1))
ds['qc_snowheight2'] = (('time',qc_snowheight2))
ds['qc_tsnow1'] = (('time',qc_tsnow1))
ds['qc_tsnow2'] = (('time',qc_tsnow2))
ds['qc_tsnow3'] = (('time',qc_tsnow3))
ds['qc_tsnow4'] = (('time',qc_tsnow4))
ds['qc_tsnow5'] = (('time',qc_tsnow5))
ds['qc_tsnow6'] = (('time',qc_tsnow6))
ds['qc_tsnow7'] = (('time',qc_tsnow7))
ds['qc_tsnow8'] = (('time',qc_tsnow8))
ds['qc_tsnow9'] = (('time',qc_tsnow9))
ds['qc_tsnow10'] = (('time',qc_tsnow10))
ds['qc_battery'] = (('time',qc_battery))
print('retrieving lat and lon...')
if station_number == 1:
temp_stn = 'gcnet_swiss'
elif station_number == 2:
temp_stn = 'gcnet_crawford'
elif station_number == 3:
temp_stn = 'gcnet_nasa-u'
elif station_number == 4:
temp_stn = 'gcnet_gits'
elif station_number == 5:
temp_stn = 'gcnet_humboldt'
elif station_number == 6:
temp_stn = 'gcnet_summit'
elif station_number == 7:
temp_stn = 'gcnet_tunu-n'
elif station_number == 8:
temp_stn = 'gcnet_dye2'
elif station_number == 9:
temp_stn = 'gcnet_jar'
elif station_number == 10:
temp_stn = 'gcnet_saddle'
elif station_number == 11:
temp_stn = 'gcnet_dome'
elif station_number == 12:
temp_stn = 'gcnet_nasa-e'
elif station_number == 13:
temp_stn = 'gcnet_cp2'
elif station_number == 14:
temp_stn = 'gcnet_ngrip'
elif station_number == 15:
temp_stn = 'gcnet_nasa-se'
elif station_number == 16:
temp_stn = 'gcnet_kar'
elif station_number == 17:
temp_stn = 'gcnet_jar2'
elif station_number == 18:
temp_stn = 'gcnet_kulu'
elif station_number == 19:
temp_stn = 'gcnet_jar3'
elif station_number == 20:
temp_stn = 'gcnet_aurora'
elif station_number == 21 or 26:
temp_stn = 'gcnet_petermann-gl'
elif station_number == 22:
temp_stn = 'gcnet_peterman-ela'
elif station_number == 23:
temp_stn = 'gcnet_neem'
elif station_number == 30:
temp_stn = 'gcnet_lar1'
elif station_number == 31:
temp_stn = 'gcnet_lar2'
elif station_number == 32:
temp_stn = 'gcnet_lar3'
latitude = station_dict.get(temp_stn)[0]
longitude = station_dict.get(temp_stn)[1]
if args.station_name:
print('Default station name overrided by user provided station name')
else:
station_name = station_dict.get(temp_stn)[2]
print('calculating hour...')
hour[:] = [int(x) for x in [round((y-int(y)),3)*hour_conversion for y in df['julian_decimal_time']]]
z = 0
while z < num_rows:
if hour[z] > last_hour:
hour[z] = 0
z += 1
print("calculating time and sza...")
tz = pytz.timezone(args.timezone)
dtime_1970 = datetime(1970,1,1)
dtime_1970 = tz.localize(dtime_1970.replace(tzinfo=None))
i = 0
while i < num_rows:
temp_dtime = datetime.strptime("%s %s %s" % (df['year'][i], int(df['julian_decimal_time'][i]), hour[i]), "%Y %j %H")
temp_dtime = tz.localize(temp_dtime.replace(tzinfo=None))
time[i] = (temp_dtime-dtime_1970).total_seconds()
time_bounds[i] = (time[i]-seconds_in_hour, time[i])
sza[i] = sunpos(temp_dtime,latitude,longitude,0)[1]
i += 1
if args.analysis:
print('calculating month and day...')
def get_month_day(year, day, one_based=False):
if one_based: # if Jan 1st is 1 instead of 0
day -= 1
dt = datetime(year, 1, 1) + timedelta(days=day)
return dt.month, dt.day
j = 0
while j < num_rows:
month[j] = get_month_day(int(df['year'][j]), int(df['julian_decimal_time'][j]), True)[0]
day[j] = get_month_day(int(df['year'][j]), int(df['julian_decimal_time'][j]), True)[1]
j += 1
ds['hour'] = (('time'),hour)
ds['month'] = (('time'),month)
ds['day'] = (('time'),day)
ds['time'] = (('time'),time)
ds['time_bounds'] = (('time', 'nbnd'),time_bounds)
ds['sza'] = (('time'),sza)
ds['station_number'] = ((),station_number)
ds['station_name'] = ((),station_name)
ds['latitude'] = ((),latitude)
ds['longitude'] = ((),longitude)
ds.attrs = {'title':'Surface Radiation Data from Greenland Climate Network', 'source':'Surface Observations', 'featureType':'timeSeries', 'institution':'Cooperative Institute for Research in Enviornmental Sciences',
'reference':'http://cires.colorado.edu/science/groups/steffen/gcnet/', 'Conventions':'CF-1.7', 'time_convention':"'time: point' variables match the time coordinate values exactly, whereas 'time: mean' variables are valid for the mean time within the time_bounds variable." + " e.g.: battery_voltage is measured once per hour at the time stored in the 'time' coordinate." + " On the other hand, temperature_tc_1 is continuously measured and then hourly-mean values are stored for each period contained in the time_bounds variable"}
ds['station_number'].attrs= {'units':'1', 'long_name':'Station Number'}
ds['year'].attrs= {'units':'1', 'long_name':'Year'}
ds['julian_decimal_time'].attrs= {'units':'decimal time', 'long_name':'Julian Decimal Time', 'note':'For each year, time starts at 1.0000 and ends at 365.9999.'}
ds['sw_down'].attrs= {'units':'watt meter-2', 'long_name':'Shortwave Flux Down', 'standard_name':'downwelling_shortwave_flux_in_air', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['sw_up'].attrs= {'units':'watt meter-2', 'long_name':'Shortwave Flux Up', 'standard_name':'upwelling_shortwave_flux_in_air', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['net_radiation'].attrs= {'units':'watt meter-2', 'long_name':'Net Radiation', 'standard_name':'surface_net_downward_radiative_flux', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['temperature_tc_1'].attrs= {'units':'kelvin', 'long_name':'TC-1 Air Temperature', 'standard_name':'air_temperature', 'note':'air temperature from TC sensor', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['temperature_tc_2'].attrs= {'units':'kelvin', 'long_name':'TC-2 Air Temperature', 'standard_name':'air_temperature', 'note':'air temperature from TC sensor', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['temperature_cs500_1'].attrs= {'units':'kelvin', 'long_name':'CS500-1 Air Temperature', 'standard_name':'air_temperature', 'note':'air temperature from CS500 sensor', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['temperature_cs500_2'].attrs= {'units':'kelvin', 'long_name':'CS500-2 Air Temperature', 'standard_name':'air_temperature', 'note':'air temperature from CS500 sensor', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['relative_humidity_1'].attrs= {'units':'1', 'long_name':'Relative Humidity 1', 'standard_name':'realtive_humidity', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['relative_humidity_2'].attrs= {'units':'1', 'long_name':'Relative Humidity 2', 'standard_name':'realtive_humidity', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['u1_wind_speed'].attrs= {'units':'meter second-1', 'long_name':'U1 Wind Speed', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['u2_wind_speed'].attrs= {'units':'meter second-1', 'long_name':'U2 Wind Speed', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['u_direction_1'].attrs= {'units':'degree', 'long_name':'U Direction 1', 'standard_name':'wind_from_direction', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['u_direction_2'].attrs= {'units':'degree', 'long_name':'U Direction 2', 'standard_name':'wind_from_direction', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['atmos_pressure'].attrs= {'units':'pascal', 'long_name':'Atmospheric Pressure', 'standard_name':'surface_air_pressure', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['snow_height_1'].attrs= {'units':'meter', 'long_name':'Snow Height 1', 'standard_name':'snow_height', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['snow_height_2'].attrs= {'units':'meter', 'long_name':'Snow Height 2', 'standard_name':'snow_height', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_01'].attrs= {'units':'kelvin', 'long_name':'T Snow 1', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_02'].attrs= {'units':'kelvin', 'long_name':'T Snow 2', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_03'].attrs= {'units':'kelvin', 'long_name':'T Snow 3', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_04'].attrs= {'units':'kelvin', 'long_name':'T Snow 4', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_05'].attrs= {'units':'kelvin', 'long_name':'T Snow 5', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_06'].attrs= {'units':'kelvin', 'long_name':'T Snow 6', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_07'].attrs= {'units':'kelvin', 'long_name':'T Snow 7', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_08'].attrs= {'units':'kelvin', 'long_name':'T Snow 8', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_09'].attrs= {'units':'kelvin', 'long_name':'T Snow 9', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['t_snow_10'].attrs= {'units':'kelvin', 'long_name':'T Snow 10', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['battery_voltage'].attrs= {'units':'volts', 'long_name':'Battery Voltage', 'standard_name':'battery_voltage', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['sw_down_max'].attrs= {'units':'watt meter-2', 'long_name':'Shortwave Flux Down Max', 'standard_name':'maximum_downwelling_shortwave_flux_in_air', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['sw_up_max'].attrs= {'units':'watt meter-2', 'long_name':'Shortwave Flux Up Max', 'standard_name':'maximum_upwelling_shortwave_flux_in_air', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['net_radiation_max'].attrs= {'units':'watt meter-2', 'long_name':'Net Radiation Max', 'standard_name':'maximum_net_radiation', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['max_air_temperature_1'].attrs= {'units':'kelvin', 'long_name':'Max Air Temperature 1', 'standard_name':'air_temperature', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['max_air_temperature_2'].attrs= {'units':'kelvin', 'long_name':'Max Air Temperature 2', 'standard_name':'air_temperature', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['min_air_temperature_1'].attrs= {'units':'kelvin', 'long_name':'Min Air Temperature 1', 'standard_name':'air_temperature', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['min_air_temperature_2'].attrs= {'units':'kelvin', 'long_name':'Min Air Temperature 2', 'standard_name':'air_temperature', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['max_windspeed_u1'].attrs= {'units':'meter second-1', 'long_name':'Max Windspeed-U1', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['max_windspeed_u2'].attrs= {'units':'meter second-1', 'long_name':'Max Windspeed-U2', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['stdev_windspeed_u1'].attrs= {'units':'meter second-1', 'long_name':'StdDev Windspeed-U1', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['stdev_windspeed_u2'].attrs= {'units':'meter second-1', 'long_name':'StdDev Windspeed-U2', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['ref_temperature'].attrs= {'units':'kelvin', 'long_name':'Reference Temperature', 'standard_name':'Need to ask network manager about long name', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['windspeed_2m'].attrs= {'units':'meter second-1', 'long_name':'Windspeed@2m', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['windspeed_10m'].attrs= {'units':'meter second-1', 'long_name':'Windspeed@10m', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['wind_sensor_height_1'].attrs= {'units':'meter', 'long_name':'Wind Sensor Height 1', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['wind_sensor_height_2'].attrs= {'units':'meter', 'long_name':'Wind Sensor Height 2', 'standard_name':'', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['albedo'].attrs= {'units':'1', 'long_name':'Albedo', 'standard_name':'surface_albedo', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['zenith_angle'].attrs= {'units':'degree', 'long_name':'Zenith Angle', 'standard_name':'solar_zenith_angle', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['qc1'].attrs= {'units':'1', 'long_name':'Quality Control variables 01-08', 'coordinates':'longitude latitude'}
ds['qc9'].attrs= {'units':'1', 'long_name':'Quality Control variables 09-16', 'coordinates':'longitude latitude'}
ds['qc17'].attrs= {'units':'1', 'long_name':'Quality Control variables 17-24', 'coordinates':'longitude latitude'}
ds['qc25'].attrs= {'units':'1', 'long_name':'Quality Control variables 25-27', 'coordinates':'longitude latitude'}
ds['qc_swdn'].attrs= {'units':'1', 'long_name':'Quality Control flag for Shortwave Flux Down'}
ds['qc_swup'].attrs= {'units':'1', 'long_name':'Quality Control flag for Shortwave Flux Up'}
ds['qc_netradiation'].attrs= {'units':'1', 'long_name':'Quality Control flag for Net Radiation'}
ds['qc_ttc1'].attrs= {'units':'1', 'long_name':'Quality Control flag for TC-1 Air Temperature'}
ds['qc_ttc2'].attrs= {'units':'1', 'long_name':'Quality Control flag for TC-2 Air Temperature'}
ds['qc_tcs1'].attrs= {'units':'1', 'long_name':'Quality Control flag for CS500-1 Air Temperature'}
ds['qc_tcs2'].attrs= {'units':'1', 'long_name':'Quality Control flag for CS500-2 Air Temperature'}
ds['qc_rh1'].attrs= {'units':'1', 'long_name':'Quality Control flag for Relative Humidity 1'}
ds['qc_rh2'].attrs= {'units':'1', 'long_name':'Quality Control flag for Relative Humidity 2'}
ds['qc_u1'].attrs= {'units':'1', 'long_name':'Quality Control flag for U1 Wind Speed'}
ds['qc_u2'].attrs= {'units':'1', 'long_name':'Quality Control flag for U2 Wind Speed'}
ds['qc_ud1'].attrs= {'units':'1', 'long_name':'Quality Control flag for U Direction 1'}
ds['qc_ud2'].attrs= {'units':'1', 'long_name':'Quality Control flag for U Direction 2'}
ds['qc_pressure'].attrs= {'units':'1', 'long_name':'Quality Control flag for Atmospheric Pressure'}
ds['qc_snowheight1'].attrs= {'units':'1', 'long_name':'Quality Control flag for Snow Height 1'}
ds['qc_snowheight2'].attrs= {'units':'1', 'long_name':'Quality Control flag for Snow Height 2'}
ds['qc_tsnow1'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 1'}
ds['qc_tsnow2'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 2'}
ds['qc_tsnow3'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 3'}
ds['qc_tsnow4'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 4'}
ds['qc_tsnow5'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 5'}
ds['qc_tsnow6'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 6'}
ds['qc_tsnow7'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 7'}
ds['qc_tsnow8'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 8'}
ds['qc_tsnow9'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 9'}
ds['qc_tsnow10'].attrs= {'units':'1', 'long_name':'Quality Control flag for T Snow 10'}
ds['qc_battery'].attrs= {'units':'1', 'long_name':'Quality Control flag for Battery Voltage'}
ds['time'].attrs= {'units':'seconds since 1970-01-01 00:00:00', 'long_name':'time of measurement', 'standard_name':'time', 'bounds':'time_bounds', 'calendar':'noleap'}
ds['sza'].attrs= {'units':'degree', 'long_name':'Solar Zenith Angle', 'standard_name':'solar_zenith_angle', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['station_name'].attrs= {'long_name':'Station Name', 'cf_role':'timeseries_id'}
ds['latitude'].attrs= {'units':'degrees_north', 'standard_name':'latitude'}
ds['longitude'].attrs= {'units':'degrees_east', 'standard_name':'longitude'}
encoding = {'julian_decimal_time': {'_FillValue': False},
'sw_down': {'_FillValue': fillvalue_double},
'sw_up': {'_FillValue': fillvalue_double},
'net_radiation': {'_FillValue': fillvalue_double},
'temperature_tc_1': {'_FillValue': fillvalue_double},
'temperature_tc_2': {'_FillValue': fillvalue_double},
'temperature_cs500_1': {'_FillValue': fillvalue_double},
'temperature_cs500_2': {'_FillValue': fillvalue_double},
'relative_humidity_1': {'_FillValue': fillvalue_double},
'relative_humidity_2': {'_FillValue': fillvalue_double},
'u1_wind_speed': {'_FillValue': fillvalue_double},
'u2_wind_speed': {'_FillValue': fillvalue_double},
'u_direction_1': {'_FillValue': fillvalue_double},
'u_direction_2': {'_FillValue': fillvalue_double},
'atmos_pressure': {'_FillValue': fillvalue_double},
'snow_height_1': {'_FillValue': fillvalue_double},
'snow_height_2': {'_FillValue': fillvalue_double},
't_snow_01': {'_FillValue': fillvalue_double},
't_snow_02': {'_FillValue': fillvalue_double},
't_snow_03': {'_FillValue': fillvalue_double},
't_snow_04': {'_FillValue': fillvalue_double},
't_snow_05': {'_FillValue': fillvalue_double},
't_snow_06': {'_FillValue': fillvalue_double},
't_snow_07': {'_FillValue': fillvalue_double},
't_snow_08': {'_FillValue': fillvalue_double},
't_snow_09': {'_FillValue': fillvalue_double},
't_snow_10': {'_FillValue': fillvalue_double},
'battery_voltage': {'_FillValue': fillvalue_double},
'sw_down_max': {'_FillValue': fillvalue_double},
'sw_up_max': {'_FillValue': fillvalue_double},
'net_radiation_max': {'_FillValue': fillvalue_double},
'max_air_temperature_1': {'_FillValue': fillvalue_double},
'max_air_temperature_2': {'_FillValue': fillvalue_double},
'min_air_temperature_1': {'_FillValue': fillvalue_double},
'min_air_temperature_2': {'_FillValue': fillvalue_double},
'max_windspeed_u1': {'_FillValue': fillvalue_double},
'max_windspeed_u2': {'_FillValue': fillvalue_double},
'stdev_windspeed_u1': {'_FillValue': fillvalue_double},
'stdev_windspeed_u2': {'_FillValue': fillvalue_double},
'ref_temperature': {'_FillValue': fillvalue_double},
'windspeed_2m': {'_FillValue': fillvalue_double},
'windspeed_10m': {'_FillValue': fillvalue_double},
'wind_sensor_height_1': {'_FillValue': fillvalue_double},
'wind_sensor_height_2': {'_FillValue': fillvalue_double},
'albedo': {'_FillValue': fillvalue_double},
'zenith_angle': {'_FillValue': fillvalue_double},
'time': {'_FillValue': False},
'time_bounds': {'_FillValue': False},
'sza': {'_FillValue': False},
'latitude': {'_FillValue': False},
'longitude': {'_FillValue': False}
}
write_data(args, ds, op_file, encoding)
def promice2nc(args, op_file, station_dict, station_name, convert_temp,
convert_press, seconds_in_hour, fillvalue_double):
header_rows = 1
convert_current = 1000
check_na = -999
column_names = [
'year', 'month', 'day', 'hour', 'day_of_year', 'day_of_century',
'air_pressure', 'air_temperature', 'air_temperature_hygroclip',
'relative_humidity_wrtwater', 'relative_humidity', 'wind_speed',
'wind_direction', 'shortwave_radiation_down',
'shortwave_radiation_down_cor', 'shortwave_radiation_up',
'shortwave_radiation_up_cor', 'albedo_theta',
'longwave_radiation_down', 'longwave_radiation_up', 'cloudcover',
'surface_temp', 'height_sensor_boom', 'height_stakes',
'depth_pressure_transducer', 'depth_pressure_transducer_cor',
'ice_temp_01', 'ice_temp_02', 'ice_temp_03', 'ice_temp_04',
'ice_temp_05', 'ice_temp_06', 'ice_temp_07', 'ice_temp_08',
'tilt_east', 'tilt_north', 'time_GPS', 'latitude_GPS', 'longitude_GPS',
'elevation', 'hor_dil_prec', 'logger_temp', 'fan_current',
'battery_voltage'
]
df = pd.read_csv(args.input_file or args.fl_in,
delim_whitespace=True,
skiprows=header_rows,
skip_blank_lines=True,
header=None,
names=column_names)
df.index.name = 'time'
df.replace(check_na, np.nan, inplace=True)
df.loc[:, [
'air_temperature', 'air_temperature_hygroclip', 'surface_temp',
'ice_temp_01', 'ice_temp_02', 'ice_temp_03', 'ice_temp_04',
'ice_temp_05', 'ice_temp_06', 'ice_temp_07', 'ice_temp_08',
'logger_temp'
]] += convert_temp
df.loc[:, ['air_pressure']] *= convert_press
df.loc[:, ['fan_current']] /= convert_current
df = df.where((pd.notnull(df)), fillvalue_double)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
# Intializing variables
num_rows = df['year'].size
time, time_bounds, sza, velocity, ice_velocity_GPS_total, ice_velocity_GPS_x, ice_velocity_GPS_y = (
[0] * num_rows for x in range(7))
print('retrieving lat and lon...')
k = os.path.basename(args.input_file or args.fl_in)
if ('EGP') in k:
temp_stn = 'promice_egp'
elif ('KAN_B') or ('Kangerlussuaq-B') in k:
temp_stn = 'promice_kanb'
elif ('KAN_L') or ('Kangerlussuaq-L') in k:
temp_stn = 'promice_kanl'
elif ('KAN_M') or ('Kangerlussuaq-M') in k:
temp_stn = 'promice_kanm'
elif ('KAN_U') or ('Kangerlussuaq-U') in k:
temp_stn = 'promice_kanu'
elif ('KPC_L') or ('KronprinsChristianland-L') in k:
temp_stn = 'promice_kpcl'
elif ('KPC_U') or ('KronprinsChristianland-U') in k:
temp_stn = 'promice_kpcu'
elif ('MIT') in k:
temp_stn = 'promice_mit'
elif ('NUK_K') or ('Nuuk-K') in k:
temp_stn = 'promice_nukk'
elif ('NUK_L') or ('Nuuk-L') in k:
temp_stn = 'promice_nukl'
elif ('NUK_N') or ('Nuuk-N') in k:
temp_stn = 'promice_nukn'
elif ('NUK_U') or ('Nuuk-U') in k:
temp_stn = 'promice_nuku'
elif ('QAS_A') or ('Qassimiut-A') in k:
temp_stn = 'promice_qasa'
elif ('QAS_L') or ('Qassimiut-L') in k:
temp_stn = 'promice_qasl'
elif ('QAS_M') or ('Qassimiut-M') in k:
temp_stn = 'promice_qasm'
elif ('QAS_U') or ('Qassimiut-U') in k:
temp_stn = 'promice_qasu'
elif ('SCO_L') or ('Scoresbysund-L') in k:
temp_stn = 'promice_scol'
elif ('SCO_U') or ('Scoresbysund-U') in k:
temp_stn = 'promice_scou'
elif ('TAS_A') or ('Tasiilaq-A') in k:
temp_stn = 'promice_tasa'
elif ('TAS_L') or ('Tasiilaq-L') in k:
temp_stn = 'promice_tasl'
elif ('TAS_U') or ('Tasiilaq-U') in k:
temp_stn = 'promice_tasu'
elif ('THU_L') or ('ThuleAirbase-L') in k:
temp_stn = 'promice_thul'
elif ('THU_U') or ('ThuleAirbase-U') in k:
temp_stn = 'promice_thuu'
elif ('UPE_L') or ('Upernavik-L') in k:
temp_stn = 'promice_upel'
elif ('UPE_U') or ('Upernavik-U') in k:
temp_stn = 'promice_upeu'
elif ('CEN') in k:
temp_stn = 'promice_cen'
latitude = (station_dict.get(temp_stn)[0])
longitude = (station_dict.get(temp_stn)[1])
if args.station_name:
print('Default station name overrided by user provided station name')
else:
station_name = station_dict.get(temp_stn)[2]
print('converting lat_GPS and lon_GPS...')
def lat_lon_gps(coords):
deg = np.floor(coords / 100)
minutes = np.floor(((coords / 100) - deg) * 100)
seconds = (((coords / 100) - deg) * 100 - minutes) * 100
return deg + minutes / 60 + seconds / 3600
# Exclude NAs
logic1 = df.latitude_GPS != fillvalue_double
logic2 = df.longitude_GPS != fillvalue_double
df1 = df[logic1]
df2 = df[logic2]
df.latitude_GPS = lat_lon_gps(df1.latitude_GPS)
df.longitude_GPS = lat_lon_gps(df2.longitude_GPS)
print('calculating time and sza...')
tz = pytz.timezone(args.timezone)
dtime_1970 = datetime(1970, 1, 1)
dtime_1970 = tz.localize(dtime_1970.replace(tzinfo=None))
i = 0
while i < num_rows:
temp_dtime = datetime(df['year'][i], df['month'][i], df['day'][i],
df['hour'][i])
temp_dtime = tz.localize(temp_dtime.replace(tzinfo=None))
time[i] = (temp_dtime - dtime_1970).total_seconds()
time_bounds[i] = (time[i], time[i] + seconds_in_hour)
sza[i] = sunpos(temp_dtime, latitude, longitude, 0)[1]
i += 1
print('calculating ice velocity...')
def ice_velocity(n, o):
m, p = 0, 1
R = 6373.0 #Approx radius of earth
while p < num_rows:
if (df['latitude_GPS'][m] == fillvalue_double
or df['longitude_GPS'][m] == fillvalue_double
or df['latitude_GPS'][n] == fillvalue_double
or df['longitude_GPS'][o] == fillvalue_double):
velocity[m] = fillvalue_double
else:
lat1 = radians(df['latitude_GPS'][m])
lon1 = radians(df['longitude_GPS'][m])
lat2 = radians(df['latitude_GPS'][n])
lon2 = radians(df['longitude_GPS'][o])
dlat = lat2 - lat1
dlon = lon2 - lon1
a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
distance = (
R * c) * 1000 #Multiplied by 1000 to convert km to meters
velocity[m] = round(
distance / seconds_in_hour, 4
) #Divided by 3600 because time change between 2 records is one hour
m += 1
n += 1
o += 1
p += 1
return velocity[:]
ice_velocity_GPS_total[:] = ice_velocity(1, 1)
ice_velocity_GPS_x[:] = ice_velocity(0, 1)
ice_velocity_GPS_y[:] = ice_velocity(1, 0)
ds['ice_velocity_GPS_total'] = (('time'), ice_velocity_GPS_total)
ds['ice_velocity_GPS_x'] = (('time'), ice_velocity_GPS_x)
ds['ice_velocity_GPS_y'] = (('time'), ice_velocity_GPS_y)
ds['time'] = (('time'), time)
ds['time_bounds'] = (('time', 'nbnd'), time_bounds)
ds['sza'] = (('time'), sza)
ds['station_name'] = ((), station_name)
ds['latitude'] = ((), latitude)
ds['longitude'] = ((), longitude)
ds.attrs = {
'title':
'Weather Station Data',
'source':
'Surface Observations',
'featureType':
'timeSeries',
'institution':
'Programme for Monitoring of the Greenland Ice Sheet',
'reference':
'http://www.promice.dk/home.html',
'Conventions':
'CF-1.7',
'time_convention':
"'time: point' variables match the time coordinate values exactly, whereas 'time: mean' variables are valid for the mean time within the time_bounds variable."
+
" e.g.: elevation is measured once per hour at the time stored in the 'time' coordinate."
+
" On the other hand, air_temperature is continuously measured and then hourly-mean values are stored for each period contained in the time_bounds variable"
}
ds['year'].attrs = {'units': '1', 'long_name': 'Year'}
ds['month'].attrs = {'units': '1', 'long_name': 'Month of Year'}
ds['day'].attrs = {'units': '1', 'long_name': 'Day of Month'}
ds['hour'].attrs = {'units': '1', 'long_name': 'Hour of Day(UTC)'}
ds['day_of_year'].attrs = {'units': '1', 'long_name': 'Day of Year'}
ds['day_of_century'].attrs = {'units': '1', 'long_name': 'Day of Century'}
ds['air_pressure'].attrs = {
'units': 'pascal',
'long_name': 'Air Pressure',
'standard_name': 'air_pressure',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['air_temperature'].attrs = {
'units': 'kelvin',
'long_name': 'Air Temperature',
'standard_name': 'air_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['air_temperature_hygroclip'].attrs = {
'units': 'kelvin',
'long_name': 'Air Temperature HygroClip',
'standard_name': 'air_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['relative_humidity_wrtwater'].attrs = {
'units': '1',
'long_name': 'Relative Humidity wrt Water',
'standard_name': 'relative_humidity',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['relative_humidity'].attrs = {
'units': '1',
'long_name': 'Relative Humidity',
'standard_name': 'relative_humidity',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['wind_speed'].attrs = {
'units': 'meter second-1',
'long_name': 'Wind Speed',
'standard_name': 'wind_speed',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['wind_direction'].attrs = {
'units': 'degree',
'long_name': 'Wind Direction',
'standard_name': 'wind_from_direction',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['shortwave_radiation_down'].attrs = {
'units': 'watt meter-2',
'long_name': 'Shortwave Radiation Down',
'standard_name': 'downwelling_shortwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['shortwave_radiation_down_cor'].attrs = {
'units': 'watt meter-2',
'long_name': 'Shortwave Radiation Down Cor',
'standard_name': 'downwelling_shortwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['shortwave_radiation_up'].attrs = {
'units': 'watt meter-2',
'long_name': 'Shortwave Radiation Up',
'standard_name': 'upwelling_shortwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['shortwave_radiation_up_cor'].attrs = {
'units': 'watt meter-2',
'long_name': 'Shortwave Radiation Up Cor',
'standard_name': 'upwelling_shortwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['albedo_theta'].attrs = {
'units': '1',
'long_name': 'Albedo_theta<70d',
'standard_name': 'surface_albedo',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['longwave_radiation_down'].attrs = {
'units': 'watt meter-2',
'long_name': 'Longwave Radiation Down',
'standard_name': 'downwelling_longwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['longwave_radiation_up'].attrs = {
'units': 'watt meter-2',
'long_name': 'Longwave Radiation Up',
'standard_name': 'upwelling_longwave_flux_in_air',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['cloudcover'].attrs = {
'units': '1',
'long_name': 'Cloud Cover',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['surface_temp'].attrs = {
'units': 'kelvin',
'long_name': 'Surface Temperature',
'standard_name': 'surface_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['height_sensor_boom'].attrs = {
'units': 'meter',
'long_name': 'Height Sensor Boom',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['height_stakes'].attrs = {
'units': 'meter',
'long_name': 'Height Stakes',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['depth_pressure_transducer'].attrs = {
'units': 'meter',
'long_name': 'Depth Pressure Transducer',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['depth_pressure_transducer_cor'].attrs = {
'units': 'meter',
'long_name': 'Depth Pressure Transducer Cor',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['ice_temp_01'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 1',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_02'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 2',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_03'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 3',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_04'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 4',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_05'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 5',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_06'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 6',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_07'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 7',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_temp_08'].attrs = {
'units': 'kelvin',
'long_name': 'Ice Temperature 8',
'standard_name': 'land_ice_temperature',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['tilt_east'].attrs = {
'units': 'degree',
'long_name': 'Tilt to East',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['tilt_north'].attrs = {
'units': 'degree',
'long_name': 'Tilt to North',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['time_GPS'].attrs = {
'units': 'UTC',
'long_name': 'Time GPS(hhmmssUTC)',
'standard_name': 'time'
}
ds['latitude_GPS'].attrs = {
'units': 'degrees_north',
'long_name': 'Latitude GPS',
'standard_name': 'latitude'
}
ds['longitude_GPS'].attrs = {
'units': 'degrees_east',
'long_name': 'Longitude GPS',
'standard_name': 'longitude'
}
ds['elevation'].attrs = {
'units': 'meter',
'long_name': 'Elevation GPS',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['hor_dil_prec'].attrs = {
'units': '1',
'long_name': 'Horizontal Dilution of Precision GPS',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['logger_temp'].attrs = {
'units': 'kelvin',
'long_name': 'Logger Temperature',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['fan_current'].attrs = {
'units': 'ampere',
'long_name': 'Fan Current',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['battery_voltage'].attrs = {
'units': 'volts',
'long_name': 'Battery Voltage',
'standard_name': 'battery_voltage',
'coordinates': 'longitude latitude',
'cell_methods': 'time: point'
}
ds['ice_velocity_GPS_total'].attrs = {
'units': 'meter second-1',
'long_name': 'Ice velocity derived from GPS Lat and Long',
'standard_name': '',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_velocity_GPS_x'].attrs = {
'units': 'meter second-1',
'long_name':
'x-component of Ice velocity derived from GPS Lat and Long',
'standard_name': 'land_ice_surface_x_velocity',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['ice_velocity_GPS_y'].attrs = {
'units': 'meter second-1',
'long_name':
'y-component of Ice velocity derived from GPS Lat and Long',
'standard_name': 'land_ice_surface_y_velocity',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['time'].attrs = {
'units': 'seconds since 1970-01-01 00:00:00',
'long_name': 'time of measurement',
'standard_name': 'time',
'bounds': 'time_bounds',
'calendar': 'noleap'
}
ds['sza'].attrs = {
'units': 'degree',
'long_name': 'Solar Zenith Angle',
'standard_name': 'solar_zenith_angle',
'coordinates': 'longitude latitude',
'cell_methods': 'time: mean'
}
ds['station_name'].attrs = {
'long_name': 'Station Name',
'cf_role': 'timeseries_id'
}
ds['latitude'].attrs = {
'units': 'degrees_north',
'standard_name': 'latitude'
}
ds['longitude'].attrs = {
'units': 'degrees_east',
'standard_name': 'longitude'
}
encoding = {
'year': {
'_FillValue': False
},
'month': {
'_FillValue': False
},
'day': {
'_FillValue': False
},
'hour': {
'_FillValue': False
},
'day_of_year': {
'_FillValue': False
},
'day_of_century': {
'_FillValue': False
},
'air_pressure': {
'_FillValue': fillvalue_double
},
'air_temperature': {
'_FillValue': fillvalue_double
},
'air_temperature_hygroclip': {
'_FillValue': fillvalue_double
},
'relative_humidity_wrtwater': {
'_FillValue': fillvalue_double
},
'relative_humidity': {
'_FillValue': fillvalue_double
},
'wind_speed': {
'_FillValue': fillvalue_double
},
'wind_direction': {
'_FillValue': fillvalue_double
},
'shortwave_radiation_down': {
'_FillValue': fillvalue_double
},
'shortwave_radiation_down_cor': {
'_FillValue': fillvalue_double
},
'shortwave_radiation_up': {
'_FillValue': fillvalue_double
},
'shortwave_radiation_up_cor': {
'_FillValue': fillvalue_double
},
'albedo_theta': {
'_FillValue': fillvalue_double
},
'longwave_radiation_down': {
'_FillValue': fillvalue_double
},
'longwave_radiation_up': {
'_FillValue': fillvalue_double
},
'cloudcover': {
'_FillValue': fillvalue_double
},
'surface_temp': {
'_FillValue': fillvalue_double
},
'height_sensor_boom': {
'_FillValue': fillvalue_double
},
'height_stakes': {
'_FillValue': fillvalue_double
},
'depth_pressure_transducer': {
'_FillValue': fillvalue_double
},
'depth_pressure_transducer_cor': {
'_FillValue': fillvalue_double
},
'ice_temp_01': {
'_FillValue': fillvalue_double
},
'ice_temp_02': {
'_FillValue': fillvalue_double
},
'ice_temp_03': {
'_FillValue': fillvalue_double
},
'ice_temp_04': {
'_FillValue': fillvalue_double
},
'ice_temp_05': {
'_FillValue': fillvalue_double
},
'ice_temp_06': {
'_FillValue': fillvalue_double
},
'ice_temp_07': {
'_FillValue': fillvalue_double
},
'ice_temp_08': {
'_FillValue': fillvalue_double
},
'tilt_east': {
'_FillValue': fillvalue_double
},
'tilt_north': {
'_FillValue': fillvalue_double
},
'time_GPS': {
'_FillValue': fillvalue_double
},
'latitude_GPS': {
'_FillValue': fillvalue_double
},
'longitude_GPS': {
'_FillValue': fillvalue_double
},
'elevation': {
'_FillValue': fillvalue_double
},
'hor_dil_prec': {
'_FillValue': fillvalue_double
},
'logger_temp': {
'_FillValue': fillvalue_double
},
'fan_current': {
'_FillValue': fillvalue_double
},
'battery_voltage': {
'_FillValue': fillvalue_double
},
'ice_velocity_GPS_total': {
'_FillValue': fillvalue_double
},
'ice_velocity_GPS_x': {
'_FillValue': fillvalue_double
},
'ice_velocity_GPS_y': {
'_FillValue': fillvalue_double
},
'time': {
'_FillValue': False
},
'time_bounds': {
'_FillValue': False
},
'sza': {
'_FillValue': False
},
'latitude': {
'_FillValue': False
},
'longitude': {
'_FillValue': False
}
}
write_data(args, ds, op_file, encoding)
def change_size(self, root_path, label_file, output_path):
times = 1
ratio = 0.2
with open(label_file) as f:
for line in f.readlines():
line = line.replace('\r', '').replace('\n', '')
print(line)
list_str = line.split(' ')
image_path = os.path.join(root_path, list_str[0])
print(image_path)
img = cv2.imread(image_path)
h, w, c = img.shape
print(h, w, c)
# img = cv2.resize(img, (img.shape[0] * times, img.shape[1] * times))
padding_h = int(h * ratio)
# image[padding:resize_h-padding, 0:image.shape[1]]
new_img = img[padding_h:h - padding_h, 0:w].copy()
if int(list_str[1]) != 4:
print('[ERROR] ' + list_str[0] + ' points not 4 !!')
cv2.circle(img, (int(float(list_str[2]) * times),
int(float(list_str[3])) * times), 3,
(0, 0, 255), -1)
cv2.circle(img, (int(float(list_str[4]) * times),
int(float(list_str[5])) * times), 3,
(0, 255, 255), -1)
cv2.circle(img, (int(float(list_str[6]) * times),
int(float(list_str[7])) * times), 3,
(255, 0, 0), -1)
cv2.circle(img, (int(float(list_str[8]) * times),
int(float(list_str[9])) * times), 3,
(0, 255, 0), -1)
cv2.imshow('check_image', img)
point_list = list()
for i in range(2, 10):
if i % 2 != 0:
point_list.append(
float('%.2f' % ((float(list_str[i]) - padding_h) *
h / new_img.shape[0])))
else:
point_list.append(float(list_str[i]))
print(point_list)
new_img = cv2.resize(new_img, (h, w))
my_img = new_img.copy()
cv2.circle(my_img, (int(float(point_list[0]) * times),
int(float(point_list[1])) * times), 3,
(0, 0, 255), -1)
cv2.circle(my_img, (int(float(point_list[2]) * times),
int(float(point_list[3])) * times), 3,
(0, 255, 255), -1)
cv2.circle(my_img, (int(float(point_list[4]) * times),
int(float(point_list[5])) * times), 3,
(255, 0, 0), -1)
cv2.circle(my_img, (int(float(point_list[6]) * times),
int(float(point_list[7])) * times), 3,
(0, 255, 0), -1)
cv2.imshow('new_image', my_img)
while True:
# 保存这张图片
k = cv2.waitKey(1) & 0xFF
if k == ord('y'):
save_path = os.path.join(output_path, list_str[0])
print(save_path)
(file_path, tempfilename) = os.path.split(save_path)
print(file_path)
mkdir_if_not_exist(file_path)
cv2.imwrite(save_path, new_img)
data = list_str[0] + " " + str(len(point_list) / 2)
for point in point_list:
data += ' ' + str(point)
data += '\n'
common.write_data(
os.path.join(output_path, 'label.txt'), data, 'a+')
break
if k == ord('n'):
break
print(
"===================================================================="
)
return
def aaws2nc(args, op_file, station_dict, station_name, convert_temp, convert_press, seconds_in_hour, fillvalue_double):
header_rows = 8
column_names = ['timestamp', 'air_temp', 'vtempdiff', 'rh', 'pressure', 'wind_dir', 'wind_spd']
df = pd.read_csv(args.input_file or args.fl_in, skiprows = header_rows, skip_blank_lines=True, header=None, names = column_names)
df.index.name = 'time'
df.loc[:,'air_temp'] += convert_temp
df.loc[:,'pressure'] *= convert_press
df = df.where((pd.notnull(df)), fillvalue_double)
ds = xr.Dataset.from_dataframe(df)
ds = ds.drop('time')
# Intializing variables
num_rows = df['timestamp'].size
time, time_bounds, sza = ([0]*num_rows for x in range(3))
print('retrieving latitude and longitude...')
f = open(args.input_file or args.fl_in)
f.readline()
for line in f:
x = str(line[12:].strip('\n'))
break
f.close()
if x == 'AGO-4':
temp_stn = 'aaws_ago4'
elif x == 'Alexander Tall Tower!':
temp_stn = 'aaws_alexander'
elif x == 'Austin':
temp_stn = 'aaws_austin'
elif x == 'Baldrick':
temp_stn = 'aaws_baldrick'
elif x == 'Bear Peninsula':
temp_stn = 'aaws_bearpeninsula'
elif x == 'Bonaparte Point':
temp_stn = 'aaws_bonapartepoint'
elif x == 'Byrd':
temp_stn = 'aaws_byrd'
elif x == 'Cape Bird':
temp_stn = 'aaws_capebird'
elif x == 'Cape Denison':
temp_stn = 'aaws_capedenison'
elif x == 'Cape Hallett':
temp_stn = 'aaws_capehallett'
elif x == 'D-10':
temp_stn = 'aaws_d10'
elif x == 'D-47':
temp_stn = 'aaws_d47'
elif x == 'D-85':
temp_stn = 'aaws_d85'
elif x == 'Dismal Island':
temp_stn = 'aaws_dismalisland'
elif x == 'Dome C II':
temp_stn = 'aaws_domecII'
elif x == 'Dome Fuji':
temp_stn = 'aaws_domefuji'
elif x == 'Elaine':
temp_stn = 'aaws_elaine'
elif x == 'Elizabeth':
temp_stn = 'aaws_elizabeth'
elif x == 'Emilia':
temp_stn = 'aaws_emilia'
elif x == 'Emma':
temp_stn = 'aaws_emma'
elif x == 'Erin':
temp_stn = 'aaws_erin'
elif x == 'Evans Knoll':
temp_stn = 'aaws_evansknoll'
elif x == 'Ferrell':
temp_stn = 'aaws_ferrell'
elif x == 'Gill':
temp_stn = 'aaws_gill'
elif x == 'Harry':
temp_stn = 'aaws_harry'
elif x == 'Henry':
temp_stn = 'aaws_henry'
elif x == 'Janet':
temp_stn = 'aaws_janet'
elif x == 'JASE2007':
temp_stn = 'aaws_jase2007'
elif x == 'Kathie':
temp_stn = 'aaws_kathie'
elif x == 'Kominko-Slade':
temp_stn = 'aaws_kominkoslade'
elif x == 'Laurie II':
temp_stn = 'aaws_laurieII'
elif x == 'Lettau':
temp_stn = 'aaws_lettau'
elif x == 'Linda':
temp_stn = 'aaws_linda'
elif x == 'Lorne':
temp_stn = 'aaws_lorne'
elif x == 'Manuela':
temp_stn = 'aaws_manuela'
elif x == 'Marble Point':
temp_stn = 'aaws_marblepoint'
elif x == 'Marble Point II':
temp_stn = 'aaws_marblepointII'
elif x == 'Margaret':
temp_stn = 'aaws_margaret'
elif x == 'Marilyn':
temp_stn = 'aaws_marilyn'
elif x == 'Minna Bluff':
temp_stn = 'aaws_minnabluff'
elif x == 'Mizuho':
temp_stn = 'aaws_mizuho'
elif x == 'Mount Siple':
temp_stn = 'aaws_mountsiple'
elif x == 'Nico':
temp_stn = 'aaws_nico'
elif x == 'PANDA-South':
temp_stn = 'aaws_pandasouth'
elif x == 'Pegasus North':
temp_stn = 'aaws_pegasusnorth'
elif x == 'Phoenix':
temp_stn = 'aaws_phoenix'
elif x == 'Port Martin':
temp_stn = 'aaws_portmartin'
elif x == 'Possession Island':
temp_stn = 'aaws_possessionisland'
elif x == 'Relay Station':
temp_stn = 'aaws_relaystation'
elif x == 'Sabrina':
temp_stn = 'aaws_sabrina'
elif x == 'Schwerdtfeger':
temp_stn = 'aaws_schwerdtfeger'
elif x == 'Siple Dome':
temp_stn = 'aaws_sipledome'
elif x == 'Theresa':
temp_stn = 'aaws_theresa'
elif x == 'Thurston Island':
temp_stn = 'aaws_thurstonisland'
elif x == 'Vito':
temp_stn = 'aaws_vito'
elif x == 'White Island':
temp_stn = 'aaws_whiteisland'
elif x == 'Whitlock':
temp_stn = 'aaws_whitlock'
elif x == 'Willie Field':
temp_stn = 'aaws_williefield'
elif x == 'Windless Bight':
temp_stn = 'aaws_windlessbight'
latitude = (station_dict.get(temp_stn)[0])
longitude = (station_dict.get(temp_stn)[1])
print('retrieving station name...')
if args.station_name:
print('Default station name overrided by user provided station name')
else:
station_name = x
print('calculating time and sza...')
tz = pytz.timezone(args.timezone)
dtime_1970 = datetime(1970,1,1)
dtime_1970 = tz.localize(dtime_1970.replace(tzinfo=None))
i = 0
with open(args.input_file or args.fl_in, "r") as infile:
for line in infile.readlines()[header_rows:]:
temp_dtime = datetime.strptime(line.strip().split(",")[0], '%Y-%m-%dT%H:%M:%SZ')
temp_dtime = tz.localize(temp_dtime.replace(tzinfo=None))
time[i] = (temp_dtime-dtime_1970).total_seconds()
time_bounds[i] = (time[i]-seconds_in_hour, time[i])
sza[i] = sunpos(temp_dtime,latitude,longitude,0)[1]
i += 1
ds['time'] = (('time'),time)
ds['time_bounds'] = (('time', 'nbnd'),time_bounds)
ds['sza'] = (('time'),sza)
ds['station_name'] = ((),station_name)
ds['latitude'] = ((),latitude)
ds['longitude'] = ((),longitude)
ds.attrs = {'source':'surface observation', 'featureType':'timeSeries', 'institution':'UW SSEC', 'reference':'https://amrc.ssec.wisc.edu/', 'Conventions':'CF-1.7', 'data_type':'q1h', 'time_convention':"'time: point' variables match the time coordinate values exactly, whereas 'time: mean' variables are valid for the mean time within the time_bounds variable." + " e.g.: air_temp is continuously measured and then hourly-mean values are stored for each period contained in the time_bounds variable"}
ds['air_temp'].attrs= {'units':'kelvin', 'long_name':'air temperature', 'standard_name':'air_temperature', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['vtempdiff'].attrs= {'units':'1', 'long_name':'vertical temperature differential', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['rh'].attrs= {'units':'1', 'long_name':'relative humidity', 'standard_name':'relative_humidity', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['pressure'].attrs= {'units':'pascal', 'long_name':'air pressure', 'standard_name':'air_pressure', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['wind_dir'].attrs= {'units':'degree', 'long_name':'wind direction', 'standard_name':'wind_from_direction', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['wind_spd'].attrs= {'units':'meter second-1', 'long_name':'wind speed', 'standard_name':'wind_speed', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['time'].attrs= {'units':'seconds since 1970-01-01 00:00:00', 'long_name':'time of measurement', 'standard_name':'time', 'bounds':'time_bounds', 'calendar':'noleap'}
ds['sza'].attrs= {'units':'degree', 'long_name':'Solar Zenith Angle', 'standard_name':'solar_zenith_angle', 'coordinates':'longitude latitude', 'cell_methods':'time: mean'}
ds['station_name'].attrs= {'long_name':'Station Name', 'cf_role':'timeseries_id'}
ds['latitude'].attrs= {'units':'degrees_north', 'standard_name':'latitude'}
ds['longitude'].attrs= {'units':'degrees_east', 'standard_name':'longitude'}
encoding = {'air_temp': {'_FillValue': fillvalue_double},
'vtempdiff': {'_FillValue': fillvalue_double},
'rh': {'_FillValue': fillvalue_double},
'pressure': {'_FillValue': fillvalue_double},
'wind_dir': {'_FillValue': fillvalue_double},
'wind_spd': {'_FillValue': fillvalue_double},
'time': {'_FillValue': False},
'time_bounds': {'_FillValue': False},
'sza': {'_FillValue': False},
'latitude': {'_FillValue': False},
'longitude': {'_FillValue': False}
}
write_data(args, ds, op_file, encoding)
