def get_menu_in_date_range(begin, end):
"""
Get menus in-between two dates (inclusive)
:param begin: A string in the format of YYYY-MM-DD specifying the start day (inclusive)
:param end: A string in the format of YYYY-MM-DD specifying the end day (inclusive)
:return: A JSON format in the form of
{"menus": [<list of JSON objects representing a menu record that also contains a list of recipe objects for that menu record>]}
"""
if not check_date(begin) or not check_date(end):
return jsonify({"error": "Dates must be in YYYY-MM-DD format"}), 400
menus = Menu().all(comparisons={"date": ["BETWEEN", [begin, end]]})
if not menus:
return jsonify({"error": "No menus between dates {} and {} found".format(begin, end)}), 404
cursor = g.db.cursor()
for menu in menus:
cursor.execute(
"SELECT *\n"
"FROM mongoose.recipes\n"
"WHERE rec_id IN (SELECT recipe_id\n"
" FROM mongoose.serves\n"
" WHERE menu_id = %s)",
(menu['id'],))
menu['recipes'] = cursor.fetchall()
cursor.close()
return jsonify({"menus": menus})
def menu_thiessen():
print("*************************************************************")
print("* Cálculo da precipitação média usando o método de Thiessen *")
print("*************************************************************")
hidroweb_dir = check_dir(input("Informe o diretório com os dados das estações do Hidroweb:\n"),'dir')
inventory = check_dir(input("Caminho completo para o inventário das estações:\n"),'file',ext='.csv')
shp = check_dir(input("Shapefile com o traçado da área de interesse:\n"),'file',ext='.shp')
poly = input("Nome do polígono da área de interesse:\n")
attr = input("Nome do atributo para seleção do polígono:\n")
buffer = input("Valor do buffer:\n")
print('Deseja informar lista com o ID dos postos de interesse?(y\n)\n')
tp = input()
while tp[0].lower() not in ['y','n','s']:
tp = input('Opção inválida. Tente novamente:\n')
if tp == 'n':
list_ids = False
else:
list_ids = check_dir(input('Informe caminho para arquivo com lista de estações:\n'),
'file',ext='.csv')
print('Data inciail:')
dates = [check_date()]
print('Data final:')
dates.append(check_date())
dir_out = check_out(input("Informe o diretório de saída:\n"))
calc_thiessen(hidroweb_dir, inventory, list_ids, shp, poly, attr, buffer,
dates, dir_out)
def get_menu_by_date(date):
"""
Get menus on a specific date
:param date: A date string in the format YYYY-MM-DD
:return: A JSON format in the form of
{"menus": [<list of JSON objects representing a menu record that also contains a list of recipe objects for that menu record>]}
"""
if not check_date(date):
return jsonify({"error": "Dates must be in YYYY-MM-DD format"}), 400
menus = Menu().find_by_attribute("date", date, limit=-1)
if not menus:
return jsonify({"error": "No menus for the date {}".format(date)}), 404
cursor = g.db.cursor()
for menu in menus:
cursor.execute(
"SELECT *\n"
"FROM mongoose.recipes\n"
"WHERE rec_id IN (SELECT recipe_id\n"
" FROM mongoose.serves\n"
" WHERE menu_id = %s)",
(menu['id'],))
menu['recipes'] = cursor.fetchall()
cursor.close()
return jsonify({"menus": menus})
def menu_post():
"""
Take in a JSON object in the form of
{
"menus":
[
{
"id": 1,
"time_of_day": <one of ('breakfast', 'lunch', 'dinner')>,
"date": <date string in the format YYYY-MM-DD>,
"recipes": [a list of numeric (integer) ids representing the ids of recipes to associate to this menu]
}, ...
]
}
Where for each entry in "menu" if there is an "id" attribute,
it is assumed that we are updating an existing menu (because we wouldn't have the id otherwise),
and the values given will update the record and its ingredients links in the database.
Otherwise, if there is no "id" for an object, the system will assume the values
given are for a new record in the database and will create a new record.
:return: A JSON object of {"menu":[<list of JSON Objects corresponding to menus updated or created in the system]}
"""
if not request.json or len(request.json) == 0:
return jsonify({"error": "No JSON supplied"}), 400
id_col = Menu.__keys__[0]
menu = Menu()
ret_val = []
if not request.json.get('menus', None):
return jsonify({"error": "Invalid schema"}), 400
for m in request.json['menus']:
menu_id = m.get(id_col, None)
# check for data validity
# Enforce a datetime format
if m.get('date', None) and not check_date(m['date']):
return jsonify({"error": "{} is not a valid date".format(m['date'])}), 400
# Enforce enums
if m.get('time_of_day', None):
if m['time_of_day'] not in Menu.__columns__['time_of_day']:
return jsonify({"error": "{} is not a valid time of day".format(m['time_of_day'])}), 400
if menu_id:
menu.find_by_id(menu_id)
menu.update(**m)
menu.flush()
else:
menu.create(**m)
if m.get('recipes', None) is not None:
try:
menu.recipes = m['recipes']
except TypeError:
return jsonify(
{"error": "Invalid data. The recipes attribute must be a list of numeric recipe ids"}), 400
ret_val.append(deepcopy(menu.data))
return jsonify({"menus": ret_val})
def add(args):
if len(args) < 3:
return print("at least one task")
with open("checklist.txt", "a") as reader:
task_date = check_date(args[3]) if len(args) > 3 else date.today()
task_id = datetime.now().strftime("%Y%m%d%H%M%S%f")
reader.write(f"{task_id},{args[2]},{task_date},n\n")
return print("task added")
def buy(product_name, buy_date, price, expiration_date):
# Check if routine is called with the right parameters
error_message = ''
if product_name is None:
error_message += "ERROR: missing argument --product_name\n"
if price is None:
error_message += "ERROR: missing argument --price\n"
if check_date(expiration_date) is None:
error_message += "ERROR: missing argument --expiration_date or wrong format <YYYY-MM-DD>\n"
if check_date(buy_date) is None:
error_message += "ERROR: missing argument --buy_date or wrong format <YYYY-MM-DD>"
# If parameters ok, add data to bought.csv file
if error_message == '':
# If bought.csv exists, read last ID else id = 0
file = pathlib.Path(csv_filename)
if file.exists():
with open(csv_filename, 'r') as f:
opened_file = f.readlines()
id = int(opened_file[-1].split(',')[0])
id = id + 1
else:
id = 0
# Write data row with ID to csv file
with open(csv_filename, mode='a', newline='', encoding='utf-8') as f:
product_writer = csv.writer(f,
delimiter=',',
quotechar="'",
quoting=csv.QUOTE_NONNUMERIC)
product_writer.writerow(
[id, product_name, buy_date, price, expiration_date])
# Action finished without problems, return 'Ok'
return 'Ok'
else:
# Error occured, return error
return error_message
def background_thread():
"""Example of how to send server generated events to clients."""
global data
while True:
socketio.sleep(0.1)
now = datetime.datetime.now()
nowstr = now.strftime("%H:%M:%S")
weather, temp = get_weather(now)
trains = get_trains(now, 10)
exchange = get_exchange()
data = dict(time=nowstr,
weather=weather,
temp=temp,
traintime=trains[0],
traintimenext=trains[1],
usd=exchange['usd'],
eur=exchange['eur'])
check_date(now)
socketio.emit('my_response', data=data, namespace='/test')
def get_data(year, month, day):
if not check_date(year, month, day):
data = {"message": "Impossible date"}
return jsonify(data), 400
path = "reports/" + year + "_events/"
noaa = NoaaReport(year, month, day, path)
try:
noaa.get_dataframe()
return Response(noaa.df.to_json(), mimetype="application/json")
except NoEventReports:
data = {"message": "No event reports."}
return jsonify(data), 500
except FileNotFoundError:
data = {"message": "File not found."}
return jsonify(data), 500
def get_menu_by_time_of_day_and_date(time_of_day, date):
"""
Get a menu given its time of day and date
:param time_of_day: A lowercase string of one of the following: ('breakfast', 'lunch', 'dinner')
:param date: A date string in the form of YYYY-MM-DD
:return: A JSON format in the form of
{"menu": {<A JSON object representing a menu record that also contains a list of recipe objects for that menu record>}}
"""
if not check_date(date):
return jsonify({"error": "Dates must be in YYYY-MM-DD format"})
if time_of_day not in Menu.__columns__['time_of_day']:
return jsonify({"error": "Time of day must be one of {}".format(Menu.__columns__['time_of_day'])}), 400
menu = Menu().find_by_attribute("date", date, limit=-1)
menu = filter(lambda x: x['time_of_day'] == time_of_day, menu)
if not menu:
return jsonify({"error": "No menu found for the time of day {} at date {}".format(time_of_day, date)}), 404
return jsonify({"menu": menu})
def sell(product_name, sell_date, price):
# Convert report date to date object
try:
sold_date = datetime.strptime(sell_date.strip("'"), '%Y-%m-%d')
except BaseException:
return 'ERROR, Wrong date format'
# Check if routine is called with the right parameters
error_message = ''
if product_name is None:
error_message += "ERROR: missing argument --product_name\n"
if check_date(sell_date) is None:
error_message += "ERROR: missing argument --sell_date or wrong format <YYYY-MM-DD>\n"
if price is None:
error_message += "ERROR: missing argument --price\n"
# If parameters ok, add data to sold.csv file
if error_message == '':
# If sold.csv exists, read last id nr else id = 0
file = pathlib.Path(csv_outputfile)
if file.exists():
with open(csv_outputfile, 'r') as f:
opened_file = f.readlines()
id = int(opened_file[-1].split(',')[0])
id = id + 1
else:
id = 0
# Get bought list
bought = read_bought(sell_date)
if id > 0:
# Read products sold and remove sold products from bought list
sold = read_sold(sell_date)
for item in sold.items():
product_sell_date = datetime.strptime(item[1][1].strip("'"),
'%Y-%m-%d')
# If sell_date is before report_date, remove item from bought
# list
if product_sell_date <= sold_date:
if item[1][0] in bought:
del bought[item[1][0]]
# Check if product available
product_id = -1
for item in bought.items():
if item[1][0] == product_name:
product_id = item[0]
break
if product_id != -1:
# Write data row with id to csv file
with open(csv_outputfile, mode='a', newline='',
encoding='utf-8') as f:
product_writer = csv.writer(f,
delimiter=',',
quotechar="'",
quoting=csv.QUOTE_NONNUMERIC)
product_writer.writerow(
[id, int(product_id), sell_date, price])
# Action finished without problems, return 'Ok'
return 'Ok'
else:
return 'ERROR: Product not in stock'
else:
# Error occured, return error
return error_message
def list_with_date(args):
return print(query_date(check_date(args[3])), end="")
def f_noise_columns(df, val):
u = df.shape[0]
return [col_name for col_name in df.columns if df[col_name].unique().shape[0] / u >= val]
number_columns = [col_name for col_name in df_X.columns if col_name.startswith('number')]
noise_columns = f_noise_columns(df_X[number_columns], 0.95)
model_config['noise_columns'] = noise_columns
print('noise_columns: {} {}'.format(len(noise_columns), noise_columns))
df_X.drop(noise_columns, axis=1, inplace=True)
df_X_test.drop(noise_columns, axis=1, inplace=True)
#
# Замена колонок string с датой на дату
datestr_columns = [col_name for col_name in df_X.columns if col_name.startswith('string')]
datestr_columns = [col_name for col_name in datestr_columns if check_date(df_X[col_name])]
print('datestr_columns: {}'.format(datestr_columns))
# Переименовываем колонки string в datetime_string
for col_name in datestr_columns:
df_X.rename(columns={col_name: 'datetime_' + col_name}, inplace=True)
df_X_test.rename(columns={col_name: 'datetime_' + col_name}, inplace=True)
# Преобразуем колонки datetime_ в тип datetime и добавляем признаки из даты
def transform_datetime_features(df):
datetime_columns = [col_name for col_name in df.columns if col_name.startswith('datetime')]
for col_name in datetime_columns:
# df[col_name] = df[col_name].apply(lambda x: parse_dt(x))
# df['number_weekday_{}'.format(col_name)] = df[col_name].apply(lambda x: x.weekday())
# df['number_month_{}'.format(col_name)] = df[col_name].apply(lambda x: x.month)
def do_gridding(suffix = "relax", start_year = defaults.START_YEAR, end_year = defaults.END_YEAR, start_month = 1, end_month = 12,
doQC = False, doQC1it = False, doQC2it = False, doQC3it = False, doSST_SLP = False,
doBC = False, doBCtotal = False, doBChgt = False, doBCscn = False, doUncert = False):
#def do_gridding(suffix = "relax", start_year = defaults.START_YEAR, end_year = defaults.END_YEAR, start_month = 1, end_month = 12, doQC = False, doSST_SLP = False, doBC = False, doUncert = False):
# end
'''
Do the gridding, first to 3hrly 1x1, then to daily 1x1 and finally monthly 1x1 and 5x5
:param str suffix: "relax" or "strict" criteria
:param int start_year: start year to process
:param int end_year: end year to process
:param int start_month: start month to process
:param int end_month: end month to process
:param bool doQC: incorporate the QC flags or not
:param bool doQC1it: incorporate the first iteration (no buddy) QC flags or not
:param bool doQC2it: incorporate the second iteration (no buddy) QC flags or not
:param bool doQC3it: incorporate the third iteration (buddy) QC flags or not
:param bool doSST_SLP: process additional variables or not
:param bool doBC: work on the bias corrected data
:param bool doBCtotal: work on the full bias corrected data
:param bool doBChgt: work on the height only bias corrected data
:param bool doBCscn: work on the screen only bias corrected data
:param bool doUncert: work on files with uncertainty information (not currently used)
:returns:
'''
# KATE modified
settings = set_paths_and_vars.set(doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn, doQC = doQC, doQC1it = doQC1it, doQC2it = doQC2it, doQC3it = doQC3it)
#settings = set_paths_and_vars.set(doBC = doBC, doQC = doQC)
# end
# KATE modified - added other BC options
# if doBC:
if doBC | doBCtotal | doBChgt | doBCscn:
# end
fields = mds.TheDelimitersExt # extended (BC)
else:
fields = mds.TheDelimitersStd # Standard
# KATE modified - added other BC options
# OBS_ORDER = utils.make_MetVars(settings.mdi, doSST_SLP = doSST_SLP, multiplier = True, doBC = doBC) # ensure that convert from raw format at writing stage with multiplier
OBS_ORDER = utils.make_MetVars(settings.mdi, doSST_SLP = doSST_SLP, multiplier = True, doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn) # ensure that convert from raw format at writing stage with multiplier
# end
# KW switching between 4 ('_strict') for climatology build and 2 for anomaly buily ('_relax') - added subscripts to files
if suffix == "relax":
N_OBS_DAY = 2 # KW ok for anomalies but this was meant to be 4 for dailies_all? and 2 for dailies_night/day?
N_OBS_FRAC_MONTH = 0.3
elif suffix == "strict":
N_OBS_DAY = 4
N_OBS_FRAC_MONTH = 0.3
# flags to check on and values to allow through
# KATE modified
if doQC1it | doQC2it:
these_flags = {"ATclim":0,"ATrep":0,"DPTclim":0,"DPTssat":0,"DPTrep":0,"DPTrepsat":0}
else:
these_flags = {"ATbud":0, "ATclim":0,"ATrep":0,"DPTbud":0,"DPTclim":0,"DPTssat":0,"DPTrep":0,"DPTrepsat":0}
#these_flags = {"ATbud":0, "ATclim":0,"ATrep":0,"DPTbud":0,"DPTclim":0,"DPTssat":0,"DPTrep":0,"DPTrepsat":0}
# end
# spin through years and months to read files
for year in np.arange(start_year, end_year + 1):
for month in np.arange(start_month, end_month + 1):
times = utils.TimeVar("time", "time since 1/{}/{} in hours".format(month, year), "hours", "time")
grid_hours = np.arange(0, 24 * calendar.monthrange(year, month)[1], DELTA_HOUR)
times.data = grid_hours
# process the monthly file
# KATE modified - added other BC options
# if doBC:
if doBC | doBCtotal | doBChgt | doBCscn:
# end
filename = "new_suite_{}{:02d}_{}_extended.txt".format(year, month, settings.OUTROOT)
else:
filename = "new_suite_{}{:02d}_{}.txt".format(year, month, settings.OUTROOT)
# KATE modified - added other BC options
# raw_platform_data, raw_obs, raw_meta, raw_qc = utils.read_qc_data(filename, settings.ICOADS_LOCATION, fields, doBC = doBC)
raw_platform_data, raw_obs, raw_meta, raw_qc = utils.read_qc_data(filename, settings.ICOADS_LOCATION, fields, doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn)
# end
# extract observation details
lats, lons, years, months, days, hours = utils.process_platform_obs(raw_platform_data)
# test dates *KW - SHOULDN'T NEED THIS - ONLY OBS PASSING DATE CHECK ARE INCLUDED*
# *RD* - hasn't run yet but will leave it in just in case of future use.
if not utils.check_date(years, year, "years", filename):
sys.exit(1)
if not utils.check_date(months, month, "months", filename):
sys.exit(1)
# KATE modified - seems to be an error with missing global name plots so have changed to settings.plots
# Choose this one to only output once per decade
#if settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
# Choose this one to output a plot for each month
if settings.plots:
#if plots and (year in [1973, 1983, 1993, 2003, 2013]):
# end
# plot the distribution of hours
import matplotlib.pyplot as plt
plt.clf()
plt.hist(hours, np.arange(-100,2500,100))
plt.ylabel("Number of observations")
plt.xlabel("Hours")
plt.xticks(np.arange(-300, 2700, 300))
plt.savefig(settings.PLOT_LOCATION + "obs_distribution_{}{:02d}_{}.png".format(year, month, suffix))
# only for a few of the variables
for variable in OBS_ORDER:
if variable.name in ["marine_air_temperature", "dew_point_temperature", "specific_humidity", "relative_humidity", "marine_air_temperature_anomalies", "dew_point_temperature_anomalies", "specific_humidity_anomalies", "relative_humidity_anomalies"]:
#plot_qc_diagnostics.values_vs_lat(variable, lats, raw_obs[:, variable.column], raw_qc, these_flags, settings.PLOT_LOCATION + "qc_actuals_{}_{}{:02d}_{}.png".format(variable.name, year, month, suffix), multiplier = variable.multiplier, doBC = doBC)
plot_qc_diagnostics.values_vs_lat_dist(variable, lats, raw_obs[:, variable.column], raw_qc, these_flags, \
settings.PLOT_LOCATION + "qc_actuals_{}_{}{:02d}_{}.png".format(variable.name, year, month, suffix), multiplier = variable.multiplier, \
# KATE modified - added other BC options
doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn)
# end
# QC sub-selection
# KATE modified - added QC iterations but also think this needs to include the bias corrected versions because the QC flags need to be applied to those too.
# Not sure what was happening previously with the doBC run - any masking to QC'd obs?
if doQC | doQC1it | doQC2it | doQC3it | doBC | doBCtotal | doBChgt | doBCscn:
#if doQC:
# end
print "Using {} as flags".format(these_flags)
# KATE modified - BC options
# mask = utils.process_qc_flags(raw_qc, these_flags, doBC = doBC)
mask = utils.process_qc_flags(raw_qc, these_flags, doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn)
# end
print "All Obs: ",len(mask)
print "Good Obs: ",len(mask[np.where(mask == 0)])
print "Bad Obs: ",len(mask[np.where(mask == 1)])
#pdb.set_trace()
complete_mask = np.zeros(raw_obs.shape)
for i in range(raw_obs.shape[1]):
complete_mask[:,i] = mask
clean_data = np.ma.masked_array(raw_obs, mask = complete_mask)
# end
else:
print "No QC flags selected"
clean_data = np.ma.masked_array(raw_obs, mask = np.zeros(raw_obs.shape))
# discretise hours
hours = utils.make_index(hours, DELTA_HOUR, multiplier = 100)
# get the hours since start of month
hours_since = ((days - 1) * 24) + (hours * DELTA_HOUR)
# discretise lats/lons
lat_index = utils.make_index(lats, DELTA_LAT, multiplier = 100)
lon_index = utils.make_index(lons, DELTA_LON, multiplier = 100)
lat_index += ((len(grid_lats)-1)/2) # and as -ve indices are unhelpful, roll by offsetting by most westward
lon_index += ((len(grid_lons)-1)/2) # or most southerly so that (0,0) is (-90,-180)
# NOTE - ALWAYS GIVING TOP-RIGHT OF BOX TO GIVE < HARD LIMIT (as opposed to <=)
# do the gridding
# extract the full grid, number of obs, and day/night flag
# KATE MEDIAN WATCH This is hard coded to doMedian (rather than settings.doMedian) - OK WITH MEDIAN HERE!!!
# KATE modified - to add settings.doMedian instead of just doMedian which seems to be consistent with the other bits and BC options
raw_month_grid, raw_month_n_obs, this_month_period = utils.grid_1by1_cam(clean_data, raw_qc, hours_since, lat_index, lon_index, \
grid_hours, grid_lats, grid_lons, OBS_ORDER, settings.mdi, doMedian = settings.doMedian, \
doBC = doBC, doBCtotal = doBCtotal, doBChgt = doBChgt, doBCscn = doBCscn)
#raw_month_grid, raw_month_n_obs, this_month_period = utils.grid_1by1_cam(clean_data, raw_qc, hours_since, lat_index, lon_index, grid_hours, grid_lats, grid_lons, OBS_ORDER, settings.mdi, doMedian = True, doBC = doBC)
# end
print "successfully read data into 1x1 3hrly grids"
# create matching array size
this_month_period = np.tile(this_month_period, (len(OBS_ORDER),1,1,1))
for period in ["all", "day", "night"]:
if period == "day":
this_month_grid = np.ma.masked_where(this_month_period == 1, raw_month_grid)
this_month_obs = np.ma.masked_where(this_month_period[0] == 1, raw_month_n_obs) # and take first slice to re-match the array size
elif period == "night":
this_month_grid = np.ma.masked_where(this_month_period == 0, raw_month_grid)
this_month_obs = np.ma.masked_where(this_month_period[0] == 0, raw_month_n_obs) # and take first slice to re-match the array size
else:
this_month_grid = copy.deepcopy(raw_month_grid)
this_month_obs = copy.deepcopy(raw_month_n_obs)
# KATE modified
# If SwitchOutput == 1 then we're in test mode - output interim files!!!
if (SwitchOutput == 1):
# have one month of gridded data.
out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_1x1_3hr_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
utils.netcdf_write(out_filename, this_month_grid, np.zeros(this_month_obs.shape), this_month_obs, OBS_ORDER, grid_lats, grid_lons, times, frequency = "H")
## have one month of gridded data.
#out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_1x1_3hr_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
#utils.netcdf_write(out_filename, this_month_grid, np.zeros(this_month_obs.shape), this_month_obs, OBS_ORDER, grid_lats, grid_lons, times, frequency = "H")
# end
# now average over time
# Dailies
daily_hours = grid_hours.reshape(-1, 24/DELTA_HOUR)
shape = this_month_grid.shape
this_month_grid = this_month_grid.reshape(shape[0], -1, 24/DELTA_HOUR, shape[2], shape[3])
this_month_obs = this_month_obs.reshape(-1, 24/DELTA_HOUR, shape[2], shape[3])
# KATE MEDIAN WATCH - settings.doMedian is generally set to True - I think we may want the MEAN HERE!!!
# KATE modified - to hard wire in MEAN here
daily_grid = np.ma.mean(this_month_grid, axis = 2)
#if settings.doMedian:
# daily_grid = np.ma.median(this_month_grid, axis = 2)
#else:
# daily_grid = np.ma.mean(this_month_grid, axis = 2)
# end
daily_grid.fill_value = settings.mdi
# filter on number of observations/day
n_hrs_per_day = np.ma.count(this_month_grid, axis = 2)
n_obs_per_day = np.ma.sum(this_month_obs, axis = 1)
if period == "all":
bad_locs = np.where(n_hrs_per_day < N_OBS_DAY) # at least 2 of possible 8 3-hourly values (6hrly data *KW OR AT LEAST 4 3HRLY OBS PRESENT*)
else:
bad_locs = np.where(n_hrs_per_day < np.floor(N_OBS_DAY / 2.)) # at least 1 of possible 8 3-hourly values (6hrly data *KW OR AT LEAST 4 3HRLY OBS PRESENT*)
daily_grid.mask[bad_locs] = True
# KATE modified - added SwitchOutput to if loop
if (SwitchOutput == 1) and settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
#if settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
# end
# plot the distribution of hours
plt.clf()
plt.hist(n_hrs_per_day.reshape(-1), bins = np.arange(-1,10), align = "left", log = True, rwidth=0.5)
if period == "all":
plt.axvline(x = N_OBS_DAY-0.5, color = "r")
else:
plt.axvline(x = np.floor(N_OBS_DAY / 2.)-0.5, color = "r")
plt.title("Number of 1x1-3hrly in each 1x1-daily grid box")
plt.xlabel("Number of 3-hrly observations (max = 8)")
plt.ylabel("Frequency (log scale)")
plt.savefig(settings.PLOT_LOCATION + "n_grids_1x1_daily_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
plt.clf()
plt.hist(n_obs_per_day.reshape(-1), bins = np.arange(-5,100,5), log = True, rwidth=0.5)
plt.title("Total number of raw observations in each 1x1 daily grid box")
plt.xlabel("Number of raw observations")
plt.ylabel("Frequency (log scale)")
plt.savefig(settings.PLOT_LOCATION + "n_obs_1x1_daily_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
# clear up memory
del this_month_grid
del this_month_obs
gc.collect()
# KATE modified
# If SwitchOutput == 1 then we're in test mode - output interim files!!!
if (SwitchOutput == 1):
# write dailies file
times.data = daily_hours[:,0]
out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_1x1_daily_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
utils.netcdf_write(out_filename, daily_grid, n_hrs_per_day[0], n_obs_per_day, OBS_ORDER, grid_lats, grid_lons, times, frequency = "D")
#times.data = daily_hours[:,0]
#out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_1x1_daily_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
#utils.netcdf_write(out_filename, daily_grid, n_hrs_per_day[0], n_obs_per_day, OBS_ORDER, grid_lats, grid_lons, times, frequency = "D")
# end
# Monthlies
times.data = daily_hours[0,0]
# KATE modified - commenting out as we don't need this anymore
# if settings.doMedian:
# monthly_grid = np.ma.median(daily_grid, axis = 1)
# else:
# monthly_grid = np.ma.mean(daily_grid, axis = 1)
#
# monthly_grid.fill_value = settings.mdi
#
# # filter on number of observations/month
# n_grids_per_month = np.ma.count(daily_grid, axis = 1)
# bad_locs = np.where(n_grids_per_month < calendar.monthrange(year, month)[1] * N_OBS_FRAC_MONTH) # 30% of possible daily values
# monthly_grid.mask[bad_locs] = True
#
# # number of raw observations
# n_obs_per_month = np.ma.sum(n_obs_per_day, axis = 0)
#
# if settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
# # plot the distribution of days
#
# plt.clf()
# plt.hist(n_obs_per_month.reshape(-1), bins = np.arange(-10,500,10), log = True, rwidth=0.5)
# plt.title("Total number of raw observations in each 1x1 monthly grid box")
# plt.xlabel("Number of raw observations")
# plt.ylabel("Frequency (log scale)")
# plt.savefig(settings.PLOT_LOCATION + "n_obs_1x1_monthly_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
#
# plt.clf()
# plt.hist(n_grids_per_month[0].reshape(-1), bins = np.arange(-2,40,2), align = "left", log = True, rwidth=0.5)
# plt.axvline(x = calendar.monthrange(year, month)[1] * N_OBS_FRAC_MONTH, color="r")
# plt.title("Total number of 1x1 daily grids in each 1x1 monthly grid")
# plt.xlabel("Number of 1x1 daily grids")
# plt.ylabel("Frequency (log scale)")
# plt.savefig(settings.PLOT_LOCATION + "n_grids_1x1_monthly_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
#
# # write monthly 1x1 file
# out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_1x1_monthly_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
# utils.netcdf_write(out_filename, monthly_grid, n_grids_per_month[0], n_obs_per_month, OBS_ORDER, grid_lats, grid_lons, times, frequency = "M")
#
# # now to re-grid to coarser resolution
# # KW # Here we may want to use the mean because its a large area but could be sparsely
# # populated with quite different climatologies so we want
# # the influence of the outliers (we've done our best to ensure these are good values)
#
# # go from monthly 1x1 to monthly 5x5 - retained as limited overhead
# monthly_5by5, monthly_5by5_n_grids, monthly_5by5_n_obs, grid5_lats, grid5_lons = utils.grid_5by5(monthly_grid, n_obs_per_month, grid_lats, grid_lons, doMedian = settings.doMedian, daily = False)
# out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_5x5_monthly_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
#
# utils.netcdf_write(out_filename, monthly_5by5, monthly_5by5_n_grids, monthly_5by5_n_obs, OBS_ORDER, grid5_lats, grid5_lons, times, frequency = "M")
#
# if settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
# # plot the distribution of days
#
# plt.clf()
# plt.hist(monthly_5by5_n_obs.reshape(-1), bins = np.arange(0,100,5), log = True, rwidth=0.5)
# plt.title("Total number of raw observations in each 5x5 monthly grid box")
# plt.xlabel("Number of raw observations")
# plt.ylabel("Frequency (log scale)")
# plt.savefig(settings.PLOT_LOCATION + "n_obs_5x5_monthly_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
#
# plt.clf()
# plt.hist(monthly_5by5_n_grids.reshape(-1), bins = np.arange(-2,30,2), align = "left", log = True, rwidth=0.5)
# plt.axvline(x = 1, color="r")
# plt.title("Total number of 1x1 monthly grids in each 5x5 monthly grid")
# plt.xlabel("Number of 1x1 monthly grids")
# plt.ylabel("Frequency (log scale)")
# plt.savefig(settings.PLOT_LOCATION + "n_grids_5x5_monthly_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
#
# # clear up memory
# del monthly_grid
# del monthly_5by5
# del monthly_5by5_n_grids
# del monthly_5by5_n_obs
# del n_grids_per_month
# del n_obs_per_month
# del n_hrs_per_day
# gc.collect()
# end
# go direct from daily 1x1 to monthly 5x5
# KATE MEDIAN WATCH - settings.doMedian is generally set to True - I think we may want the MEAN HERE!!!
# KATE modified - to hard wire in MEAN here
monthly_5by5, monthly_5by5_n_grids, monthly_5by5_n_obs, grid5_lats, grid5_lons = utils.grid_5by5(daily_grid, n_obs_per_day, grid_lats, grid_lons, doMedian = False, daily = True)
#monthly_5by5, monthly_5by5_n_grids, monthly_5by5_n_obs, grid5_lats, grid5_lons = utils.grid_5by5(daily_grid, n_obs_per_day, grid_lats, grid_lons, doMedian = settings.doMedian, daily = True)
# end
out_filename = settings.DATA_LOCATION + settings.OUTROOT + "_5x5_monthly_from_daily_{}{:02d}_{}_{}.nc".format(year, month, period, suffix)
utils.netcdf_write(out_filename, monthly_5by5, monthly_5by5_n_grids, monthly_5by5_n_obs, OBS_ORDER, grid5_lats, grid5_lons, times, frequency = "M")
if settings.plots and (year in [1973, 1983, 1993, 2003, 2013]):
# plot the distribution of days
plt.clf()
plt.hist(monthly_5by5_n_obs.reshape(-1), bins = np.arange(-10,1000,10), log = True, rwidth=0.5)
plt.title("Total number of raw observations in each 5x5 monthly grid box")
plt.xlabel("Number of raw observations")
plt.ylabel("Frequency (log scale)")
plt.savefig(settings.PLOT_LOCATION + "n_obs_5x5_monthly_from_daily_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
plt.clf()
plt.hist(monthly_5by5_n_grids.reshape(-1), bins = np.arange(-5,100,5), align = "left", log = True, rwidth=0.5)
plt.axvline(x = (0.3 * daily_grid.shape[0]), color="r")
plt.title("Total number of 1x1 daily grids in each 5x5 monthly grid")
plt.xlabel("Number of 1x1 daily grids")
plt.ylabel("Frequency (log scale)")
plt.savefig(settings.PLOT_LOCATION + "n_grids_5x5_monthly_from_daily_{}{:02d}_{}_{}.png".format(year, month, period, suffix))
del daily_grid
del monthly_5by5
del n_obs_per_day
del monthly_5by5_n_grids
del monthly_5by5_n_obs
gc.collect()
return # do_gridding
