def get_time_and_sza(args, dataframe, longitude, latitude): """Calculate additional time related variables""" dtime_1970, tz = common.time_common(args.tz) year1900 = False dtime_1900 = tz.localize(datetime(1900, 1, 1).replace(tzinfo=None)) if dataframe['year'][0] < 1970: dtime_1970 = dtime_1900 year1900 = True num_rows = dataframe['year'].size time, time_bounds, sza, day_of_year = ([0] * num_rows for _ in range(4)) for idx in range(num_rows): keys = ('year', 'month', 'day', 'hour', 'minute') dtime = datetime(*[dataframe[k][idx] for k in keys]) dtime = tz.localize(dtime.replace(tzinfo=None)) day_of_year[idx] = dtime.timetuple().tm_yday time[idx] = (dtime - dtime_1970).total_seconds() time_bounds[idx] = (time[idx], time[idx]) dtime = datetime.utcfromtimestamp(time[idx]) sza[idx] = sunposition.sunpos(dtime, latitude, longitude, 0)[1] return time, time_bounds, sza, day_of_year, year1900
def get_time_and_sza(args, dataframe, longitude, latitude): """Calculate additional time related variables""" dtime_1970, tz = common.time_common(args.tz) num_rows = dataframe['year'].size time, time_bounds, sza, az = ([0] * num_rows for _ in range(4)) dates = [] for idx in range(num_rows): keys = ('year', 'month', 'day', 'hour') dtime = datetime(*[dataframe[k][idx] for k in keys]) dtime = tz.localize(dtime.replace(tzinfo=None)) time[idx] = (dtime - dtime_1970).total_seconds() time_bounds[idx] = (time[idx], time[idx] + common.seconds_in_hour) # Each timestamp is average of current and next hour values i.e. value at hour=5 is average of hour=5 and hour=6 # Our 'time' variable will represent values at half-hour i.e. 5.5 in above case, so add 30 minutes to all. time[idx] = time[idx] + common.seconds_in_half_hour dtime = datetime.utcfromtimestamp(time[idx]) dates.append(int(dtime.date().strftime("%Y%m%d"))) solar_angles = sunposition.sunpos(dtime, latitude, longitude, 0) az[idx] = solar_angles[0] sza[idx] = solar_angles[1] first_date = min(dates) last_date = max(dates) return time, time_bounds, sza, az, first_date, last_date
def get_time_and_sza(args, dataframe, longitude, latitude, sub_type): """Calculate additional time related variables""" seconds_in_30min = 30*60 seconds_in_15min = 15*60 dtime_1970, tz = common.time_common(args.tz) num_rows = dataframe['year'].size month, day, minutes, time, time_bounds, sza, az = ([0] * num_rows for _ in range(7)) hour = (dataframe['hour_mult_100']/100).astype(int) temp_dtime = pd.to_datetime(dataframe['year']*1000 + dataframe['day_of_year'].astype(int), format='%Y%j') dataframe['hour'] = hour dataframe['dtime'] = temp_dtime dataframe['dtime'] = pd.to_datetime(dataframe.dtime) dataframe['dtime'] = [tz.localize(i.replace(tzinfo=None)) for i in dataframe['dtime']] dataframe['dtime'] += pd.to_timedelta(dataframe.hour, unit='h') if sub_type == 'imau/ant': # Each timestamp is average of previous and current hour values i.e. value at hour=5 is average of hour=4 and 5 # Our 'time' variable will represent values at half-hour i.e. 4.5 in above case, so subtract 30 minutes from all time = (dataframe['dtime'] - dtime_1970) / np.timedelta64(1, 's') - seconds_in_30min time_bounds = [(i-seconds_in_30min, i+seconds_in_30min) for i in time] elif sub_type == 'imau/grl': minutes = (((dataframe['hour_mult_100']/100) % 1) * 100).astype(int) dataframe['minutes'] = minutes dataframe['dtime'] += pd.to_timedelta(dataframe.minutes, unit='m') # Each timestamp is average of previous and current half-hour values i.e. value at hr=5 is average of 4.5 and 5 # Our 'time' variable will represent values at half of 30 min i.e. 4.75, so subtract 15 minutes from all time = (dataframe['dtime'] - dtime_1970) / np.timedelta64(1, 's') - seconds_in_15min time_bounds = [(i-seconds_in_15min, i+seconds_in_15min) for i in time] month = pd.DatetimeIndex(dataframe['dtime']).month.values day = pd.DatetimeIndex(dataframe['dtime']).day.values dates = list(pd.DatetimeIndex(dataframe['dtime']).date) dates = [int(d.strftime("%Y%m%d")) for d in dates] first_date = min(dates) last_date = max(dates) for idx in range(num_rows): solar_angles = sunposition.sunpos(dataframe['dtime'][idx], latitude, longitude, 0) az[idx] = solar_angles[0] sza[idx] = solar_angles[1] return month, day, hour, minutes, time, time_bounds, sza, az, first_date, last_date
def get_time_and_sza(args, dataframe, longitude, latitude): """Calculate additional time related variables""" dtime_1970, tz = common.time_common(args.tz) num_rows = dataframe['year'].size sza, az = ([0] * num_rows for _ in range(2)) hour_conversion = 100 / 4 last_hour = 23 hour = dataframe['julian_decimal_time'] hour = [round(i - int(i), 3) * hour_conversion for i in hour] hour = [int(h) if int(h) <= last_hour else 0 for h in hour] temp_dtime = pd.to_datetime(dataframe['year']*1000 + dataframe['julian_decimal_time'].astype(int), format='%Y%j') dataframe['hour'] = hour dataframe['dtime'] = temp_dtime dataframe['dtime'] = pd.to_datetime(dataframe.dtime) dataframe['dtime'] += pd.to_timedelta(dataframe.hour, unit='h') # Each timestamp is average of previous and current hour values i.e. value at hour=5 is average of hour=4 and hour=5 # Our 'time' variable will represent values at half-hour i.e. 4.5 in above case, so subtract 30 minutes from all. dataframe['dtime'] -= pd.to_timedelta(common.seconds_in_half_hour, unit='s') dataframe['dtime'] = [tz.localize(i.replace(tzinfo=None)) for i in dataframe['dtime']] # Set timezone time = (dataframe['dtime'] - dtime_1970) / np.timedelta64(1, 's') # Seconds since 1970 time_bounds = [(i-common.seconds_in_half_hour, i+common.seconds_in_half_hour) for i in time] month = pd.DatetimeIndex(dataframe['dtime']).month.values day = pd.DatetimeIndex(dataframe['dtime']).day.values minutes = pd.DatetimeIndex(dataframe['dtime']).minute.values dates = list(pd.DatetimeIndex(dataframe['dtime']).date) dates = [int(d.strftime("%Y%m%d")) for d in dates] first_date = min(dates) last_date = max(dates) for idx in range(num_rows): solar_angles = sunposition.sunpos(dataframe['dtime'][idx], latitude, longitude, 0) az[idx] = solar_angles[0] sza[idx] = solar_angles[1] return month, day, hour, minutes, time, time_bounds, sza, az, first_date, last_date
def main(dataset, latitude, longitude, clr_df, args): ddr = 0.25 rho = 0.8 smallest_double = 2.2250738585072014e-308 dtime_1970, tz = common.time_common(args.tz) clrprd_file = clr_df # Combine date, start_hour and end_hour into a single string, e.g. 20080103_16_23 clrprd = [(str(x) + '_' + str(y) + '_' + str(z)) for x, y, z in zip( clrprd_file['date'].tolist(), clrprd_file['start_hour'].tolist(), clrprd_file['end_hour'].tolist())] hours = list(range(24)) half_hours = (list(np.arange(0, 24, 0.5))) ds = dataset.drop( 'time_bounds' ) # Drop time_bounds dimension so that we don't have double entries of same data df = ds.to_dataframe() # Convert to dataframe date_hour = [datetime.fromtimestamp(i, tz) for i in df.index.values] # Index is seconds since 1970 dates = [i.date() for i in date_hour] # Get dates df['dates'] = dates # Add as new column # Create new dataframe to store tilt_direction and tilt_angle tilt_df = pd.DataFrame(index=dates, columns=['tilt_direction', 'tilt_angle']) lat = latitude lon = longitude # Drop 'time' as index to do proper indexing for station_name df.reset_index(level=['time'], inplace=True) stn_name = df['station_name'][0] # Replace fillvalue with nan for calculations df[['fsds']] = df[['fsds']].replace(common.fillvalue_float, np.nan) jaws_path = 'http://jaws.ess.uci.edu/jaws/rigb_data/' dir_rrtm = 'rrtm-airx3std/' sfx = '.rrtm.nc' if args.merra: dir_rrtm = 'rrtm-merra/' rrtm_file = get_rrtm_file(jaws_path, dir_rrtm, stn_name, sfx) if rrtm_file: rrtm_df = get_rrtm_df(stn_name, sfx, rrtm_file) else: # If no AIRS RRTM file, try MERRA RRTM file dir_rrtm = 'rrtm-merra/' rrtm_file = get_rrtm_file(jaws_path, dir_rrtm, stn_name, sfx) if rrtm_file: rrtm_df = get_rrtm_df(stn_name, sfx, rrtm_file) else: print( 'ERROR: RRTM data not available for this station. Please report it on github.com/jaws/jaws/issues' ) os._exit(1) start_time = time.time() ######################################################################################## # Tilt Angle and Tilt Direction Calculations # # ########PART-1############# # for line in clrprd: clrdate = line.split('_')[0] clrhr_start = int(line.split('_')[1]) clrhr_end = int(line.split('_')[2]) clrhr_end = clrhr_end + 1 # To make sure we include the last hour when slicing the data year = int(clrdate[:4]) month = int(clrdate[4:6]) day = int(clrdate[6:]) current_date_hour = datetime(year, month, day).date() fsds_rrtm = rrtm_df.loc[str(year) + '-' + str(month) + '-' + str(day):str(year) + '-' + str(month) + '-' + str(day)]['fsds'].values.tolist() if fsds_rrtm: if args.dbg_lvl > 6: print(clrdate) else: current_time = time.time() if (current_time - start_time) > 10 * 60: print('Still working...') start_time = current_time else: continue # Subset dataframe df_sub = df[df.dates == current_date_hour] fsds_jaws_nonmsng = df_sub['fsds'].dropna().tolist() indexMissingJAWS = np.where(df_sub['fsds'].isna()) indexMissingJAWS = [a for b in indexMissingJAWS for a in b] # Convert to list hours_nonmsng = np.where(df_sub['fsds'].notnull()) hours_nonmsng = [a for b in hours_nonmsng for a in b] # Convert to list hours_nonmsng = [i + 0.5 for i in hours_nonmsng] # Half-hour values # Interpolate fsds and sza for half-hour values if len(fsds_jaws_nonmsng) < 2: if args.dbg_lvl > 6: print("Skipping this day as there is only 1 value of fsds") continue else: fsds_intrp = CubicSpline(hours_nonmsng, fsds_jaws_nonmsng, extrapolate=True)(half_hours) fsds_intrp = [a for a in fsds_intrp] # Convert to list # Calculate azimuth angle az = [] sza = [] for hour in hours: dtime = datetime(year, month, day, hour, 0) az.append(sunposition.sunpos(dtime, lat, lon, 0)[0]) sza.append(sunposition.sunpos(dtime, lat, lon, 0)[1]) dtime = datetime(year, month, day, hour, 30) az.append(sunposition.sunpos(dtime, lat, lon, 0)[0]) sza.append(sunposition.sunpos(dtime, lat, lon, 0)[1]) az = [(i - 180) for i in az] alpha = [(90 - i) for i in sza] beta = list(np.arange(0.25, 45.25, 0.25)) sza_noon = [np.cos(np.radians(i)) for i in sza] # Check if measured solar noon time > true solar noon time if fsds_intrp.index(max(fsds_intrp)) > sza_noon.index(max(sza_noon)): aw = list(np.arange(0, 180, 0.25)) else: aw = list(np.arange(-179.75, 0.25, 0.25)) az = deg_to_rad(az) alpha = deg_to_rad(alpha) beta = deg_to_rad(beta) aw = deg_to_rad(aw) # Make pairs of aw,beta pairs = [] for i in aw: for j in beta: pairs.append(tuple((i, j))) # Find all possible pairs using correct fsds possible_pairs = [] daily_avg_diff = [] best_pairs = [] fsds_possiblepair_dict = {} for pair in pairs: count = 0 cos_i = [] fsds_correct = [] while count < len(alpha): cos_i.append((np.cos(alpha[count]) * np.cos(az[count] - pair[0]) * np.sin(pair[1]) + (np.sin(alpha[count]) * np.cos(pair[1])))) nmr = fsds_intrp[count] * (np.sin(alpha[count]) + ddr) dnmr = cos_i[count] + (ddr * (1 + np.cos(pair[1])) / 2.) + (rho * (np.sin(alpha[count]) + ddr) * (1 - np.cos(pair[1])) / 2.) if dnmr == 0: dnmr = smallest_double fsds_correct.append(nmr / dnmr) count += 1 if (abs( cos_i.index(max(cos_i)) - fsds_intrp.index(max(fsds_intrp))) <= 1 and abs( fsds_correct.index(max(fsds_correct)) - sza_noon.index(max(sza_noon))) <= 1): possible_pairs.append(pair) fsds_correct_half = fsds_correct[1::2] fsds_possiblepair_dict[pair] = fsds_correct_half for msng_idx in indexMissingJAWS: try: fsds_correct_half.pop(msng_idx) except: common.log(args, 9, 'Warning: missing index fsds_correct_half') try: fsds_rrtm.pop(msng_idx) except: common.log(args, 9, 'Warning: missing index fsds_rrtm') diff = [ abs(x - y) for x, y in zip(fsds_correct_half[clrhr_start:clrhr_end], fsds_rrtm[clrhr_start:clrhr_end]) ] daily_avg_diff.append(np.nanmean(diff)) # ########PART-2############# # dailyavg_possiblepair_dict = dict(zip(daily_avg_diff, possible_pairs)) if not dailyavg_possiblepair_dict.keys(): continue # Skip day if no possible pair else: if min(dailyavg_possiblepair_dict.keys()) <= 50: for val in dailyavg_possiblepair_dict.keys(): if val <= min(dailyavg_possiblepair_dict.keys()) + 5: best_pairs.append(dailyavg_possiblepair_dict.get(val)) # ########PART-3############# # fsds_bestpair_dict = {k: fsds_possiblepair_dict[k] for k in best_pairs} bestpair_dailyavg_dict = dict((bp, [ key for (key, value) in dailyavg_possiblepair_dict.items() if value == bp ]) for bp in best_pairs) num_spikes = [] for pair in fsds_bestpair_dict: fsds_correct_top = fsds_bestpair_dict[pair] counter = 0 spike_hrs = 0 diff_top = [ abs(x - y) for x, y in zip(fsds_correct_top[clrhr_start:clrhr_end], fsds_rrtm[clrhr_start:clrhr_end]) ] fsds_rrtm_10 = [ ij * 0.1 for ij in fsds_rrtm[clrhr_start:clrhr_end] ] for val in diff_top: if diff_top[counter] > fsds_rrtm_10[counter]: spike_hrs += 1 counter += 1 num_spikes.append((spike_hrs, bestpair_dailyavg_dict[pair])) try: top_pair = best_pairs[num_spikes.index(min(num_spikes))] tilt_df.at[current_date_hour, 'tilt_direction'] = top_pair[0] tilt_df.at[current_date_hour, 'tilt_angle'] = top_pair[1] except: common.log(args, 9, 'Warning: no top pair found') continue # Skip day if no top pair ######################################################################################## tilt_df['tilt_direction'] = pd.to_numeric(tilt_df['tilt_direction'], errors='coerce') tilt_df['tilt_angle'] = pd.to_numeric(tilt_df['tilt_angle'], errors='coerce') tilt_df = tilt_df.interpolate( limit_direction='both') # Interpolate missing values tilt_direction_values = tilt_df['tilt_direction'].tolist() tilt_angle_values = tilt_df['tilt_angle'].tolist() tilt_direction_values = rad_to_deg(tilt_direction_values) dataset[ 'tilt_direction_raw'] = 'time', tilt_direction_values # Raw values to be used in fsds_adjust script # Change tilt_direction to 0 pointing north. These values will be in output netCDF file tilt_direction_values = [270 - d for d in tilt_direction_values] tilt_direction_values = [ d - 360 if d > 360 else d for d in tilt_direction_values ] tilt_angle_values = rad_to_deg(tilt_angle_values) # Add tilt_direction and tilt_angle to output file dataset['tilt_direction'] = 'time', tilt_direction_values dataset['tilt_angle'] = 'time', tilt_angle_values try: # Remove downloaded rrtm_df file os.remove(stn_name + sfx) except: # Windows pass return dataset