def find_files(st_uni, s_files):
## Detect stations
top_header('Main Menu > Automatic Detection > Solar Data')
print('\t\tPlease select on station option: \n')
print('\t\tNumber of stations found: \t', len(st_uni))
print('\t\tNumber of files: \t\t', len(s_files))
print('\n')
count = -1
for f in st_uni:
count = count + 1
num_fils = []
for fis in s_files:
if (f in fis.name):
num_fils.append(fis)
print("\t\t [%s]" % count + f + ' files -> ' + str(len(num_fils)))
while True:
try:
ans_st = int(input("\n\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_st > count:
print("\t\t Wrong selection.")
continue
station_files = []
for fff in s_files:
if (st_uni[ans_st] in fff.name):
station_files.append(fff)
return st_uni[ans_st], station_files
break
def pre_processing_menu():
top_header('Main Menu > Preprocessing Mode')
print('\t\tPlease select operational mode: ')
choice = input("""
0: Download Data
1: Generate Historical Data
2: Translate Historical Data
B: Back
Q: Quit
Please enter your choice: """)
if choice == "Download Data" or choice =="0":
download_stations()
elif choice == "Generate Historical Data" or choice =="1":
historic_generate()
elif choice == "Translate Historical Data" or choice =="2":
translate_historical()
elif choice == "B" or choice =="b":
mainMenu()
elif choice=="Q" or choice=="q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
pre_processing_menu()
def translate_menu():
top_header('Main Menu > Translate Mode')
print('\t\tPlease select operational mode: ')
choice = input("""
0: Translate Anemometric Data
1: Translate and Split Solar Data
2: Translate Sky Camera Data
B: Back
Q: Quit
Please enter your choice: """)
if choice == "Translate Anemometric Data" or choice =="0":
anemomectric()
elif choice == "Translate and Split Solar Data" or choice =="1":
split_data()
elif choice == "Translate Sky Camera Data" or choice =="2":
skycamera()
elif choice == "B" or choice =="b":
mainMenu()
elif choice=="Q" or choice=="q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
translate_menu()
def load_data_type():
top_header('Main Menu > Automatic Detection')
print('\t\tPlease select an option: ')
print('\t\tPath dir: ', path_)
choice = input("""
1: Detect Solarimetric Data
2: Detect Meteorological Data
3: Detect Anemometric Data
4: Detect Sky Camera Data
5: Set Data PATH
Q: Quit
Please enter your choice: """)
if choice == "Detect Solarimetric Data" or choice == "1":
detec_solar()
elif choice == "Detect Meteorological Data" or choice == "2":
detect_met()
elif choice == "Detect Anemometric Data" or choice == "3":
detect_ane()
elif choice == "Detect Sky Camera Data" or choice == "4":
detect_sky()
elif choice == "Set Data PATH" or choice == "5":
set_path()
elif choice == "Q" or choice == "q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
load_data_type()
def split_data():
top_header('Main Menu > Solarimetric > Split Data')
print('\t\tPlease select an option\n')
files = load_stations()
process_files(files)
def connection():
global user, password, ip
top_header(('Main Menu > Format Data > Download Data from FTP'))
print('\t\tFTP Config:')
print('\t\tIP: ', ip)
print('\t\tUser:'******'\t\tPassword:'******'\t\tPlease select an option: ')
choice = input("""
0: Connect and Download
1: Set IP
2: Set User
3: Set Password
B: Back
Q: Quit
Please enter your choice: """)
if choice == "Connect" or choice == "0":
print('Connecting....')
try:
ftp = FTP(ip)
ftp.login(user, password)
time.sleep(2)
print('Connection established!')
list_stations(ftp)
except:
time.sleep(2)
print('Connection failed')
print('Please try again!')
input()
connection()
elif choice == "Set IP" or choice == "1":
ip = input("\t\tIP: ")
connection()
elif choice == "Set User" or choice == "2":
user = input("\t\tIP: ")
connection()
elif choice == "Set Password:"******"3":
password = input("\t\tPassword: "******"B" or choice == "b":
format_new_data()
elif choice == "Q" or choice == "q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
connection()
def menu_qualify():
top_header('Qualify SONDA Data->')
path_to_qualify = load_stations_03(load_config()[0]['QUALI_IN'])
out_path = load_config()[0]['QUALI_OUT']
web_path = load_config()[0]['WEB_OUT']
if type(path_to_qualify) == str:
files = {p.resolve() for p in Path(path_to_qualify).rglob("**/*" ) if p.suffix in [EXTENSTION]}
else:
files = []
for f2 in path_to_qualify:
f3 = {p.resolve() for p in Path(f2).rglob("**/*" ) if p.suffix in [EXTENSTION]}
for f4 in f3:
files.append(Path(f4))
process_files(files,path_to_qualify,out_path,web_path)
def format_new_data():
top_header('Main Menu > Format New Data')
print('\t\tPlease select an option')
choice = input("""
0: Download from FTP
1: Offline format
Q: Quit
Please enter your choice: """)
if choice == "Download from FTP" or choice == "0":
connection()
elif choice == "Offline format" or choice == "1":
format_data()
elif choice == "Q" or choice == "q":
sys.exit
else:
print("You must only select either 1 or 2")
print("Please try again")
def list_stations(ftp):
top_header(
('Main Menu > Format Data > Download from FTP > Select station'))
print('\t\tPlease select an option: \n')
## FTP Dir
directory = "/labgama/redesonda/"
## Add STATIONS
lt_stations = ['sms', 'chp', 'orn']
# direc_st = []
# for station in ftp.nlst(directory):
# if "." in station:
# pass
# else:
# lt_stations.append(station[-3:])
# direc_st.append(station)
lt_stations = [x.upper() for x in lt_stations]
count = -1
for f in lt_stations:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
lt_stations = [x.lower() for x in lt_stations]
down_dir = directory + lt_stations[ans_file] + '/data/'
selected_st = lt_stations[ans_file]
download_data(down_dir, ftp, selected_st)
break
def load_stations_04(path):
top_header('Select type of Data')
file_names = [fn for fn in listdir(path) if not fn.startswith('.')]
count = -1
for f in file_names:
count = count + 1
print ("\t\t [%s]" % count + f)
while True:
try:
print()
ans_file = int(input("\t\t Select Type of Data: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print ("\t\t Wrong selection.")
continue
station = path + '/' + file_names[ans_file]
return load_stations_05(station)
def download_stations():
top_header('Main Menu > Preprocessing Mode > Download Data')
stations, ftp_con = connection()
### Station
count = -1
for f in stations:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
files_dir = load_config()[0]['FTP_DIR'] + stations[ans_file] + '/data/'
download_files(files_dir, ftp_con, stations[ans_file])
break
def mainMenu():
top_header('Main Menu')
print('\t\tPlease select operational mode: ')
choice = input("""
0: Preprocessing Mode
1: Translate Mode
2: Qualify Mode
Q: Quit
Please enter your choice: """)
if choice == "0: Preprocessing Mode" or choice =="0":
pre_processing_menu()
elif choice == "1: Translate Mode" or choice =="1":
translate_menu()
elif choice == "2: Qualify Mode" or choice =="2":
menu_qualify()
elif choice=="Q" or choice=="q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
mainMenu()
def load_stations_03(path):
top_header('Qualify Data - Load Stations')
file_names = [fn for fn in listdir(path) if not fn.startswith('.')]
file_names.append('ALL')
count = -1
for f in file_names:
count = count + 1
print ("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
if ans_file > count:
print ("\t\t Wrong selection.")
continue
except:
print("\t\t Wrong selection")
continue
if 'ALL' in file_names[ans_file]:
types = ['Meteorologicos','Solarimétricos']
f_file = int(input("\t\t Select type of Data: \n\t\t[0] Meteorologicos\n\t\t[1] Solarimetricos"))
if f_file == 0:
fnames = [path+fn+'/Meteorologicos/' for fn in listdir(path) if not fn.startswith('.')]
if f_file == 1:
fnames = [path+fn+'/Solarimetricos/' for fn in listdir(path) if not fn.startswith('.')]
all_paths = []
for fff in fnames:
for ff2 in listdir(fff):
if not ff2.startswith('.'):
all_paths.append(fff+ff2)
return all_paths
else:
station = path + file_names[ans_file]
return load_stations_04(station)
def find_fyears(stat_, sfiles_):
top_header('Main Menu > Automatic Detection > Solar Data > Station: ' +
str(stat_))
print('\t\tPlease select on year to process: \n')
print('\n')
## Filter years
yrs = []
for f in sfiles_:
yrs.append(f.name[4:8])
st_yrs = np.unique(np.array(yrs))
count = -1
for f in st_yrs:
count = count + 1
num_fils = []
for fis in sfiles_:
if (f in fis.name):
num_fils.append(fis)
print("\t\t [%s]" % count + f + ' files -> ' + str(len(num_fils)))
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
path = st_yrs[ans_file]
yearFiles = []
for ffy in sfiles_:
if (st_yrs[ans_file] in ffy.name):
yearFiles.append(ffy)
return yearFiles, path
break
def open_file(select_file, station, year, file):
met_header = load_config()[0]['MET_INPUT']
solar_header = load_config()[0]['SOLAR_INPUT']
##OUTPUT HEADER
met_out_header = load_config()[0]['MET_HEADER']
sol_out_header = load_config()[0]['SOLAR_HEADER']
top_header('Main > Preprocessing > Translate Historical > ' +
str(station) + ' > ' + str(year) + ' > ' + str(file))
print('\t\tPlease select one file to translate file: ')
if 'MD' in file:
header_in = met_header
header_out = met_out_header
if 'SD' in file:
header_in = solar_header
header_out = sol_out_header
if 'TD' in file:
header_in = None
header_out = None
# print(header_in[1:])
# print(header_out)
df = pd.read_csv(select_file, sep=",")
## SELECT ONLY COLUMNS INPUT
df = df[header_in[1:]]
## IGNORE MULTINDEX INTO HISTORICAL DATA
df = df.iloc[1:]
print(df)
print('aqui')
print(load_config()[0]['FORMATED_OUT'] + str(station) + '/' + str(year))
def load_files(path):
top_header(path)
years_ = [fn for fn in listdir(path) if fn.startswith('2')]
years_.sort()
count = -1
for f in years_:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
path = path + '/' + years_[ans_file] + '/'
files = get_files(path)
break
return files
def historic_generate():
operation_dir = load_config()[0]['OPERATIONAL_IN']
## SET DEBUG DIR
logging.basicConfig(filename=load_config()[0]['DEBUG_DIR'] +
'historical_debug.txt',
filemode='a',
format='\nProcess Date %(asctime)s \n %(message)s\n',
datefmt='%d-%b-%y %H:%M:%S',
level=os.environ.get("LOGLEVEL", "INFO"))
top_header('Main > Preprocessing > Generate Historical')
print('\t\tPlease select one stations to generate historical data: ')
operational_stations = [
fn for fn in listdir(operation_dir) if not fn.startswith('.')
]
if len(operational_stations) == 0:
print('There is no data to be formatted')
input('Press Enter to return')
# pre_processing_menu()
# return None
count = -1
for f in operational_stations:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
selected_st = operation_dir + operational_stations[ans_file] + '/'
break
## SELECT TYPE OF DATA
top_header('Main > Preprocessing > Generate Historical > ' +
str(operational_stations[ans_file]).upper())
print('\t\tPlease select type of data to generate historical data: ')
dataTypes = ['MD', 'SD', 'TD', '50', '25', '10']
countT = -1
for f in dataTypes:
countT = countT + 1
print("\t\t [%s]" % countT + f)
while True:
try:
ans_type = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_type > countT:
print("\t\t Wrong selection.")
continue
selected_file = operational_stations[ans_file].upper() + '_' + str(
dataTypes[ans_type]) + '.DAT'
break
### DATA TYPES
if operational_stations[ans_file] == 'sms':
## OPEN DATA
df = pd.read_csv(selected_st + selected_file,
sep=",",
header=None,
skiprows=4,
skipinitialspace=False)
df1 = df.copy()
head0 = pd.read_csv(selected_st + selected_file,
sep=",",
header=None,
nrows=1)
head1 = pd.read_csv(selected_st + selected_file,
sep=",",
header=None,
skiprows=1,
nrows=1)
head2 = pd.read_csv(selected_st + selected_file,
sep=",",
header=None,
skiprows=3,
nrows=1)
head0 = head0.iloc[0].values
head1 = head1.iloc[0].values
head2 = head2.iloc[0].values
df1[0] = pd.to_datetime(df1[0], format='%Y-%m-%d %H:%M:%S')
## SELECT TIMESTAMP TO PROCESSE
top_header('Main > Preprocessing > Generate Historical > ' +
str(operational_stations[ans_file]).upper() + ' > ' +
str(dataTypes[ans_type]))
print('\t\tPlease select type of data to generate historical data: ')
## AVAIBLE YEARS
years = df1[0].dt.year.unique()
countY = -1
for f in years:
countY = countY + 1
print("\t\t [%s]" % countY + str(f))
while True:
try:
ans_year = int(input("\t\t Select Year: "))
except:
print("\t\t Wrong selection")
continue
if ans_year > countY:
print("\t\t Wrong selection.")
continue
selected_year = years[ans_year]
break
df1 = df1.set_index(0)
months = df1.loc[str(selected_year)]
months = months.reset_index()
months = months[0].dt.strftime('%m').unique()
top_header('Main > Preprocessing > Generate Historical > ' +
str(operational_stations[ans_file]).upper() + ' > ' +
str(dataTypes[ans_type]) + ' > ' + str(selected_year))
print('\t\tPlease select type of data to generate historical data: ')
countM = -1
for f in months:
countM = countM + 1
print("\t\t [%s]" % countM + str(f))
while True:
try:
ans_month = int(input("\t\t Select Month: "))
except:
print("\t\t Wrong selection")
continue
if ans_month > countM:
print("\t\t Wrong selection.")
continue
selected_month = months[ans_month]
break
top_header('Main > Preprocessing > Generate Historical > ' +
str(operational_stations[ans_file]).upper() + ' > ' +
str(dataTypes[ans_type]) + ' > ' + str(selected_year) +
' > ' + str(selected_month))
#SELECTED TO GENERATE
df1 = df1.loc[str(selected_year) + '-' + str(selected_month)]
## TIME INTERVAL VERIFICATION
df1 = df1.sort_index(ascending=True)
# GET TIMES STRING
max_time = df1.index.max()
min_time = df1.index.min()
## FIND INDEX INTO ORIGINAL DATAFRAME
idx_min = df.loc[df[0] == str(min_time)].index.values[0]
idx_max = df.loc[df[0] == str(max_time)].index.values[0]
## LOC BETWEEN IDX
locked_df_chk = df.loc[idx_min:idx_max]
## FINAL DF TO COMPARE
final_df = locked_df_chk.copy()
## MOUNT INDEX OF DATES ACORDING TYPE OF DATA
if dataTypes[ans_type] == 'MD' or dataTypes[
ans_type] == '10' or dataTypes[ans_type] == '25' or dataTypes[
ans_type] == '50':
freqc = '10min'
if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD':
freqc = '1min'
## PASS COLUMN TO DATETIME
locked_df_chk[0] = pd.to_datetime(locked_df_chk[0],
format='%Y-%m-%d %H:%M:%S')
## MULTIINDEX
mux = []
for i in range(len(head1)):
mux.append([str(head1[i]).lower(), str(head2[i]).lower()])
mux = pd.MultiIndex.from_tuples(mux)
## GET ID OF STATION
id_st = locked_df_chk[2].values[0]
## DETECT non-existent SENSOR
non_sens_col = []
object_type_c = locked_df_chk.select_dtypes(include=['object'])
for c in object_type_c:
detc_mean = object_type_c[c].astype(float).mean()
if str(detc_mean) == 'nan':
non_sens_col.append(c)
## DESAPROVE WITH TIME INTERVAL
## DETECT TYPE OF FILE
if dataTypes[ans_type] == 'MD' or dataTypes[
ans_type] == '10' or dataTypes[ans_type] == '25' or dataTypes[
ans_type] == '50':
t_delta = pd.Timedelta(minutes=10)
if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD':
t_delta = pd.Timedelta(minutes=1)
## Generate all month days to compare
year_month = locked_df_chk[0].dt.strftime('%Y-%m').values[0]
# print(pd.Timestamp(year_month) + pd.offsets.MonthEnd(1) + pd.Timedelta(hours=24) - t_delta)
month_generated = pd.date_range(
start=pd.Timestamp(year_month),
end=pd.Timestamp(year_month) + pd.offsets.MonthEnd(1) +
pd.Timedelta(hours=24) - t_delta, # <-- 2018-08-31 with MonthEnd
freq=freqc)
## CHECK DUPLICAT IN TIMESTAMP COLUMN
times_dup = locked_df_chk[locked_df_chk.duplicated([0], keep=False)]
group_tdup = times_dup.groupby(0)
## RESOLVE CHOKE TIME STAMP
idx_first = []
for g, gdftum in group_tdup:
# print(gdftum)
idx_groups = gdftum.index
for idxgg in range(len(idx_groups)):
pass_idx = idx_groups[idxgg] - 1
if dataTypes[ans_type] == 'MD' or dataTypes[
ans_type] == '10' or dataTypes[
ans_type] == '25' or dataTypes[ans_type] == '50':
t_delta = pd.Timedelta(minutes=10)
locked_df_chk.loc[
idx_groups[idxgg],
0] = locked_df_chk.loc[pass_idx][0] + t_delta
if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD':
t_delta = pd.Timedelta(minutes=1)
locked_df_chk.loc[
idx_groups[idxgg],
0] = locked_df_chk.loc[pass_idx][0] + t_delta
## CHECK DUPLICAT IN TIMESTAMP COLUMN
locked_df_chk = locked_df_chk.drop_duplicates(subset=0)
## SET INDEX COLUMN OF TIME STAMP
locked_df_chk = locked_df_chk.set_index(0)
## CHEACK DUPLICATED IN ALL COLUMNS
locked_df_chk = locked_df_chk.drop_duplicates(keep='first')
# FILL
locked_df_chk = locked_df_chk.reindex(month_generated, fill_value=0)
## DESAPROVE WITH TIME INTERVAL
check_t_interval = locked_df_chk
check_t_interval = check_t_interval[min_time:max_time]
# ## LIMITE TIME DETAL
lim_delta = pd.Timedelta(minutes=50)
totalDelta = t_delta
for i, row in check_t_interval.iterrows():
if np.all(row[6:-1].values == 0):
totalDelta = totalDelta + t_delta
else:
last_ro = i
totalDelta = pd.Timedelta(minutes=0)
if totalDelta >= lim_delta:
print(
'Failed to generate file due to a longer time sequence of failures greater than ',
lim_delta, '\n')
print('')
fail = check_t_interval[last_ro:i + t_delta]
# fail[6:-1] = 3333
fail = fail.reset_index()
fail.columns = mux
print(fail)
return None
## ADD NON-EXISTENT VALUES
locked_df_chk[non_sens_col] = 5555
idx_values = locked_df_chk.loc[locked_df_chk[2] == 0].index
## ADD YEAR
locked_df_chk.loc[idx_values,
3] = locked_df_chk.loc[idx_values].index.strftime(
'%Y').values
## ADD JULIAN DAY
locked_df_chk.loc[idx_values,
4] = locked_df_chk.loc[idx_values].index.strftime(
'%j').values
## ADD MINUTE
for hm in idx_values:
s1 = hm.strftime('%d/%m/%Y 00:00')
s2 = hm.strftime('%d/%m/%Y %H:%M')
s1 = datetime.strptime(s1, '%d/%m/%Y %H:%M')
s2 = datetime.strptime(s2, '%d/%m/%Y %H:%M')
differenc = s2 - s1
minutes = divmod(differenc.seconds, 60)
locked_df_chk.loc[hm, 5] = minutes[0]
locked_df_chk[2] = id_st
## DATETIME COLUMNS + NON_SENSOR
dt_non_se = []
dt_non_se.append(2)
dt_non_se.append(3)
dt_non_se.append(4)
dt_non_se = dt_non_se + non_sens_col
### ADD VALUES 3333,
diff_columns = np.setdiff1d(locked_df_chk.columns.values, dt_non_se)
locked_df_chk.loc[idx_values, diff_columns] = 3333
# print(locked_df_chk['2020-09-09 13:30':'2020-09-09 17:20'].head(50))
## SAVE
if len(locked_df_chk) > 0:
## SAVE PROCESS
output_dir = load_config()[0]['HISTORICAL_OUT'] + str(
operational_stations[ans_file]).upper() + '/' + str(
selected_year) + '/'
output_file_name = str(operational_stations[ans_file]).upper(
) + '_' + str(selected_year) + '_' + str(
locked_df_chk.index[0].strftime('%j')) + '_a_' + str(
locked_df_chk.index[-1].strftime('%j')) + '_' + str(
dataTypes[ans_type]) + '.dat'
## COUNT VALUES
lost_counter = 0
nosen_counter = 0
for i, row in locked_df_chk.iterrows():
## COUNTER FOR LOST FILES
if 3333 in row.values[:]:
lost_counter = lost_counter
lost_counter += 1
## COUNTER FOR LOST FILES
if 5555 in row.values[:]:
nosen_counter = nosen_counter
nosen_counter += 1
## RESET INDEX
locked_df_chk = locked_df_chk.reset_index()
# ADD MULTIINDEX
locked_df_chk.columns = mux
# CREATE DIR
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
## CREATE VERSION DIRS
pathlib.Path(output_dir + '/versions').mkdir(parents=True,
exist_ok=True)
file__ = output_dir + output_file_name
# ## CHEACK IF FILES EXISTS
if os.path.isfile(file__):
warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) +
' \nLOST 3333 ROWS:-> ' + str(lost_counter) +
'\nNO SENSOR 5555 ROWS:\n' + str(nosen_counter) +
'\n')
logging.warning(warningmsg)
# print(warningmsg)
# print(locked_df_chk.head())
## CHECK LAST VERSION
if len(os.listdir(output_dir + '/versions')) == 0:
shutil.move(
file__,
output_dir + '/versions/' + output_file_name + '.v01')
## CREATE FILE
locked_df_chk.to_csv(file__, index=False)
warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) +
' File version(0) \nLOST 3333 ROWS:-> ' +
str(lost_counter) +
'\nNO SENSOR 5555 ROWS:\n' +
str(nosen_counter) + '\n')
logging.warning(warningmsg)
else:
versions = [
fn for fn in listdir(output_dir + '/versions/')
if not fn.startswith('.')
]
shutil.move(
file__, output_dir + '/versions/' + output_file_name +
'v0' + str((int(versions[-1][-2:]) + 1)))
warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) +
' File version(' +
str(int(versions[-1][-2:]) + 1) +
') \nLOST 3333 ROWS:-> ' +
str(lost_counter) +
'\nNO SENSOR 5555 ROWS:\n' +
str(nosen_counter) + '\n')
logging.warning(warningmsg)
print(warningmsg)
## CREATE FILE
locked_df_chk.to_csv(file__, index=False)
print(locked_df_chk)
else:
warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) +
' File version(0) \nLOST 3333 ROWS:-> ' +
str(lost_counter) + '\nNO SENSOR 5555 ROWS:\n' +
str(nosen_counter) + '\n')
logging.warning(warningmsg)
## CREATE FILE
print(warningmsg)
print(locked_df_chk)
locked_df_chk.to_csv(file__, index=False)
## UPLOAD RESULTS
ver_file_names = [
fn for fn in listdir(output_dir + str('/versions'))
if not fn.startswith('.')
]
if len(ver_file_names) > 0:
last_file_version = sorted(ver_file_names)[-1]
else:
last_file_version = None
file_to_upload = file__
print('\t\tUpload files to FTP: ')
choice = input("""
(Y) - Yes
(N) - No
Please enter your choice: """)
if choice == "Y" or choice == "y":
connection(file_to_upload, operational_stations[ans_file],
selected_year, output_file_name, last_file_version,
operational_stations[ans_file])
elif choice == "N" or choice == "n":
sys.exit
elif choice == "Q" or choice == "q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
mainMenu()
def skycamera():
top_header('Main Menu > Sky Camera')
print('\t\tPlease select an option')
load_stations_02()
def download_files(directory, connection, station):
top_header('Main Menu > Preprocessing Mode > Download Data > ' +
str(station))
print('\t\tPlease select type of the data to download: ')
choice = input("""
0: All
1: Meteorological - MD
2: Solarimetric - SD
3: Anemometric - 10
4: Anemometric - 25
5: Anemometric - 50
B: Back
Q: Quit
Please enter your choice: """)
if choice == "All" or choice == "0":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + str(station)).mkdir(
parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + str(station))
print(config_file[0]['OPERATIONAL_IN'] + str(station))
connection.cwd(directory)
for filename in connection.nlst('*.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "Meteorological" or choice == "1":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' +
str(station)).mkdir(parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station))
connection.cwd(directory)
for filename in connection.nlst('*MD.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "Solarimetric" or choice == "2":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' +
str(station)).mkdir(parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station))
connection.cwd(directory)
for filename in connection.nlst('*SD.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "Anemometric10" or choice == "3":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' +
str(station)).mkdir(parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station))
connection.cwd(directory)
for filename in connection.nlst('*10.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "Anemometric25" or choice == "4":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' +
str(station)).mkdir(parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station))
connection.cwd(directory)
for filename in connection.nlst('*25.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "Anemometric50" or choice == "5":
pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' +
str(station)).mkdir(parents=True, exist_ok=True)
os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station))
connection.cwd(directory)
for filename in connection.nlst('*50.DAT'):
fhandle = open(filename, 'wb')
print('\t\tDownloading... ' + filename)
connection.retrbinary('RETR ' + filename, fhandle.write)
fhandle.close()
print('All files has been downloaded!')
elif choice == "b" or choice == "B":
download_stations()
elif choice == "Q" or choice == "q":
sys.exit
else:
print("You must only select one option")
print("Please try again")
download_files()
def translate_historical():
operation_dir = load_config()[0]['HISTORICAL_OUT']
top_header('Main > Preprocessing > Translate Historical')
print('\t\tPlease select one stations to translate historical data: ')
historical_pats = [
fn for fn in listdir(operation_dir) if not fn.startswith('.')
]
if len(historical_pats) == 0:
print('There is no data to be formatted')
input('Press Enter to return')
## SELECT STATION
count = -1
for f in historical_pats:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file > count:
print("\t\t Wrong selection.")
continue
selected_st = operation_dir + historical_pats[ans_file] + '/'
top_header('Main > Preprocessing > Translate Historical > ' +
str(historical_pats[ans_file]))
print('\t\tPlease select one year file: ')
select_year = [
fn for fn in listdir(selected_st) if not fn.startswith('.')
]
## SELECT FILE
count = -1
for f in select_year:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_year = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_year > count:
print("\t\t Wrong selection.")
continue
selected_year = selected_st + select_year[ans_year] + '/'
top_header('Main > Preprocessing > Translate Historical > ' +
str(historical_pats[ans_file]) + ' > ' +
select_year[ans_year])
print('\t\tPlease select one file to translate file: ')
select_files = [
fn for fn in listdir(selected_year)
if not fn.startswith('.') and '.dat' in fn
]
## FOR FILE
count = -1
for f in select_files:
count = count + 1
print("\t\t [%s]" % count + f)
while True:
try:
ans_file_ = int(input("\t\t Select Station: "))
except:
print("\t\t Wrong selection")
continue
if ans_file_ > count:
print("\t\t Wrong selection.")
continue
selected_file = selected_year + select_files[ans_file_]
open_file(selected_file, historical_pats[ans_file],
select_year[ans_year], select_files[ans_file_])
break
break
break