def find_files(st_uni, s_files): ## Detect stations top_header('Main Menu > Automatic Detection > Solar Data') print('\t\tPlease select on station option: \n') print('\t\tNumber of stations found: \t', len(st_uni)) print('\t\tNumber of files: \t\t', len(s_files)) print('\n') count = -1 for f in st_uni: count = count + 1 num_fils = [] for fis in s_files: if (f in fis.name): num_fils.append(fis) print("\t\t [%s]" % count + f + ' files -> ' + str(len(num_fils))) while True: try: ans_st = int(input("\n\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_st > count: print("\t\t Wrong selection.") continue station_files = [] for fff in s_files: if (st_uni[ans_st] in fff.name): station_files.append(fff) return st_uni[ans_st], station_files break
def pre_processing_menu(): top_header('Main Menu > Preprocessing Mode') print('\t\tPlease select operational mode: ') choice = input(""" 0: Download Data 1: Generate Historical Data 2: Translate Historical Data B: Back Q: Quit Please enter your choice: """) if choice == "Download Data" or choice =="0": download_stations() elif choice == "Generate Historical Data" or choice =="1": historic_generate() elif choice == "Translate Historical Data" or choice =="2": translate_historical() elif choice == "B" or choice =="b": mainMenu() elif choice=="Q" or choice=="q": sys.exit else: print("You must only select one option") print("Please try again") pre_processing_menu()
def translate_menu(): top_header('Main Menu > Translate Mode') print('\t\tPlease select operational mode: ') choice = input(""" 0: Translate Anemometric Data 1: Translate and Split Solar Data 2: Translate Sky Camera Data B: Back Q: Quit Please enter your choice: """) if choice == "Translate Anemometric Data" or choice =="0": anemomectric() elif choice == "Translate and Split Solar Data" or choice =="1": split_data() elif choice == "Translate Sky Camera Data" or choice =="2": skycamera() elif choice == "B" or choice =="b": mainMenu() elif choice=="Q" or choice=="q": sys.exit else: print("You must only select one option") print("Please try again") translate_menu()
def load_data_type(): top_header('Main Menu > Automatic Detection') print('\t\tPlease select an option: ') print('\t\tPath dir: ', path_) choice = input(""" 1: Detect Solarimetric Data 2: Detect Meteorological Data 3: Detect Anemometric Data 4: Detect Sky Camera Data 5: Set Data PATH Q: Quit Please enter your choice: """) if choice == "Detect Solarimetric Data" or choice == "1": detec_solar() elif choice == "Detect Meteorological Data" or choice == "2": detect_met() elif choice == "Detect Anemometric Data" or choice == "3": detect_ane() elif choice == "Detect Sky Camera Data" or choice == "4": detect_sky() elif choice == "Set Data PATH" or choice == "5": set_path() elif choice == "Q" or choice == "q": sys.exit else: print("You must only select one option") print("Please try again") load_data_type()
def split_data(): top_header('Main Menu > Solarimetric > Split Data') print('\t\tPlease select an option\n') files = load_stations() process_files(files)
def connection(): global user, password, ip top_header(('Main Menu > Format Data > Download Data from FTP')) print('\t\tFTP Config:') print('\t\tIP: ', ip) print('\t\tUser:'******'\t\tPassword:'******'\t\tPlease select an option: ') choice = input(""" 0: Connect and Download 1: Set IP 2: Set User 3: Set Password B: Back Q: Quit Please enter your choice: """) if choice == "Connect" or choice == "0": print('Connecting....') try: ftp = FTP(ip) ftp.login(user, password) time.sleep(2) print('Connection established!') list_stations(ftp) except: time.sleep(2) print('Connection failed') print('Please try again!') input() connection() elif choice == "Set IP" or choice == "1": ip = input("\t\tIP: ") connection() elif choice == "Set User" or choice == "2": user = input("\t\tIP: ") connection() elif choice == "Set Password:"******"3": password = input("\t\tPassword: "******"B" or choice == "b": format_new_data() elif choice == "Q" or choice == "q": sys.exit else: print("You must only select one option") print("Please try again") connection()
def menu_qualify(): top_header('Qualify SONDA Data->') path_to_qualify = load_stations_03(load_config()[0]['QUALI_IN']) out_path = load_config()[0]['QUALI_OUT'] web_path = load_config()[0]['WEB_OUT'] if type(path_to_qualify) == str: files = {p.resolve() for p in Path(path_to_qualify).rglob("**/*" ) if p.suffix in [EXTENSTION]} else: files = [] for f2 in path_to_qualify: f3 = {p.resolve() for p in Path(f2).rglob("**/*" ) if p.suffix in [EXTENSTION]} for f4 in f3: files.append(Path(f4)) process_files(files,path_to_qualify,out_path,web_path)
def format_new_data(): top_header('Main Menu > Format New Data') print('\t\tPlease select an option') choice = input(""" 0: Download from FTP 1: Offline format Q: Quit Please enter your choice: """) if choice == "Download from FTP" or choice == "0": connection() elif choice == "Offline format" or choice == "1": format_data() elif choice == "Q" or choice == "q": sys.exit else: print("You must only select either 1 or 2") print("Please try again")
def list_stations(ftp): top_header( ('Main Menu > Format Data > Download from FTP > Select station')) print('\t\tPlease select an option: \n') ## FTP Dir directory = "/labgama/redesonda/" ## Add STATIONS lt_stations = ['sms', 'chp', 'orn'] # direc_st = [] # for station in ftp.nlst(directory): # if "." in station: # pass # else: # lt_stations.append(station[-3:]) # direc_st.append(station) lt_stations = [x.upper() for x in lt_stations] count = -1 for f in lt_stations: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue lt_stations = [x.lower() for x in lt_stations] down_dir = directory + lt_stations[ans_file] + '/data/' selected_st = lt_stations[ans_file] download_data(down_dir, ftp, selected_st) break
def load_stations_04(path): top_header('Select type of Data') file_names = [fn for fn in listdir(path) if not fn.startswith('.')] count = -1 for f in file_names: count = count + 1 print ("\t\t [%s]" % count + f) while True: try: print() ans_file = int(input("\t\t Select Type of Data: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print ("\t\t Wrong selection.") continue station = path + '/' + file_names[ans_file] return load_stations_05(station)
def download_stations(): top_header('Main Menu > Preprocessing Mode > Download Data') stations, ftp_con = connection() ### Station count = -1 for f in stations: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue files_dir = load_config()[0]['FTP_DIR'] + stations[ans_file] + '/data/' download_files(files_dir, ftp_con, stations[ans_file]) break
def mainMenu(): top_header('Main Menu') print('\t\tPlease select operational mode: ') choice = input(""" 0: Preprocessing Mode 1: Translate Mode 2: Qualify Mode Q: Quit Please enter your choice: """) if choice == "0: Preprocessing Mode" or choice =="0": pre_processing_menu() elif choice == "1: Translate Mode" or choice =="1": translate_menu() elif choice == "2: Qualify Mode" or choice =="2": menu_qualify() elif choice=="Q" or choice=="q": sys.exit else: print("You must only select one option") print("Please try again") mainMenu()
def load_stations_03(path): top_header('Qualify Data - Load Stations') file_names = [fn for fn in listdir(path) if not fn.startswith('.')] file_names.append('ALL') count = -1 for f in file_names: count = count + 1 print ("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) if ans_file > count: print ("\t\t Wrong selection.") continue except: print("\t\t Wrong selection") continue if 'ALL' in file_names[ans_file]: types = ['Meteorologicos','Solarimétricos'] f_file = int(input("\t\t Select type of Data: \n\t\t[0] Meteorologicos\n\t\t[1] Solarimetricos")) if f_file == 0: fnames = [path+fn+'/Meteorologicos/' for fn in listdir(path) if not fn.startswith('.')] if f_file == 1: fnames = [path+fn+'/Solarimetricos/' for fn in listdir(path) if not fn.startswith('.')] all_paths = [] for fff in fnames: for ff2 in listdir(fff): if not ff2.startswith('.'): all_paths.append(fff+ff2) return all_paths else: station = path + file_names[ans_file] return load_stations_04(station)
def find_fyears(stat_, sfiles_): top_header('Main Menu > Automatic Detection > Solar Data > Station: ' + str(stat_)) print('\t\tPlease select on year to process: \n') print('\n') ## Filter years yrs = [] for f in sfiles_: yrs.append(f.name[4:8]) st_yrs = np.unique(np.array(yrs)) count = -1 for f in st_yrs: count = count + 1 num_fils = [] for fis in sfiles_: if (f in fis.name): num_fils.append(fis) print("\t\t [%s]" % count + f + ' files -> ' + str(len(num_fils))) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue path = st_yrs[ans_file] yearFiles = [] for ffy in sfiles_: if (st_yrs[ans_file] in ffy.name): yearFiles.append(ffy) return yearFiles, path break
def open_file(select_file, station, year, file): met_header = load_config()[0]['MET_INPUT'] solar_header = load_config()[0]['SOLAR_INPUT'] ##OUTPUT HEADER met_out_header = load_config()[0]['MET_HEADER'] sol_out_header = load_config()[0]['SOLAR_HEADER'] top_header('Main > Preprocessing > Translate Historical > ' + str(station) + ' > ' + str(year) + ' > ' + str(file)) print('\t\tPlease select one file to translate file: ') if 'MD' in file: header_in = met_header header_out = met_out_header if 'SD' in file: header_in = solar_header header_out = sol_out_header if 'TD' in file: header_in = None header_out = None # print(header_in[1:]) # print(header_out) df = pd.read_csv(select_file, sep=",") ## SELECT ONLY COLUMNS INPUT df = df[header_in[1:]] ## IGNORE MULTINDEX INTO HISTORICAL DATA df = df.iloc[1:] print(df) print('aqui') print(load_config()[0]['FORMATED_OUT'] + str(station) + '/' + str(year))
def load_files(path): top_header(path) years_ = [fn for fn in listdir(path) if fn.startswith('2')] years_.sort() count = -1 for f in years_: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue path = path + '/' + years_[ans_file] + '/' files = get_files(path) break return files
def historic_generate(): operation_dir = load_config()[0]['OPERATIONAL_IN'] ## SET DEBUG DIR logging.basicConfig(filename=load_config()[0]['DEBUG_DIR'] + 'historical_debug.txt', filemode='a', format='\nProcess Date %(asctime)s \n %(message)s\n', datefmt='%d-%b-%y %H:%M:%S', level=os.environ.get("LOGLEVEL", "INFO")) top_header('Main > Preprocessing > Generate Historical') print('\t\tPlease select one stations to generate historical data: ') operational_stations = [ fn for fn in listdir(operation_dir) if not fn.startswith('.') ] if len(operational_stations) == 0: print('There is no data to be formatted') input('Press Enter to return') # pre_processing_menu() # return None count = -1 for f in operational_stations: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue selected_st = operation_dir + operational_stations[ans_file] + '/' break ## SELECT TYPE OF DATA top_header('Main > Preprocessing > Generate Historical > ' + str(operational_stations[ans_file]).upper()) print('\t\tPlease select type of data to generate historical data: ') dataTypes = ['MD', 'SD', 'TD', '50', '25', '10'] countT = -1 for f in dataTypes: countT = countT + 1 print("\t\t [%s]" % countT + f) while True: try: ans_type = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_type > countT: print("\t\t Wrong selection.") continue selected_file = operational_stations[ans_file].upper() + '_' + str( dataTypes[ans_type]) + '.DAT' break ### DATA TYPES if operational_stations[ans_file] == 'sms': ## OPEN DATA df = pd.read_csv(selected_st + selected_file, sep=",", header=None, skiprows=4, skipinitialspace=False) df1 = df.copy() head0 = pd.read_csv(selected_st + selected_file, sep=",", header=None, nrows=1) head1 = pd.read_csv(selected_st + selected_file, sep=",", header=None, skiprows=1, nrows=1) head2 = pd.read_csv(selected_st + selected_file, sep=",", header=None, skiprows=3, nrows=1) head0 = head0.iloc[0].values head1 = head1.iloc[0].values head2 = head2.iloc[0].values df1[0] = pd.to_datetime(df1[0], format='%Y-%m-%d %H:%M:%S') ## SELECT TIMESTAMP TO PROCESSE top_header('Main > Preprocessing > Generate Historical > ' + str(operational_stations[ans_file]).upper() + ' > ' + str(dataTypes[ans_type])) print('\t\tPlease select type of data to generate historical data: ') ## AVAIBLE YEARS years = df1[0].dt.year.unique() countY = -1 for f in years: countY = countY + 1 print("\t\t [%s]" % countY + str(f)) while True: try: ans_year = int(input("\t\t Select Year: ")) except: print("\t\t Wrong selection") continue if ans_year > countY: print("\t\t Wrong selection.") continue selected_year = years[ans_year] break df1 = df1.set_index(0) months = df1.loc[str(selected_year)] months = months.reset_index() months = months[0].dt.strftime('%m').unique() top_header('Main > Preprocessing > Generate Historical > ' + str(operational_stations[ans_file]).upper() + ' > ' + str(dataTypes[ans_type]) + ' > ' + str(selected_year)) print('\t\tPlease select type of data to generate historical data: ') countM = -1 for f in months: countM = countM + 1 print("\t\t [%s]" % countM + str(f)) while True: try: ans_month = int(input("\t\t Select Month: ")) except: print("\t\t Wrong selection") continue if ans_month > countM: print("\t\t Wrong selection.") continue selected_month = months[ans_month] break top_header('Main > Preprocessing > Generate Historical > ' + str(operational_stations[ans_file]).upper() + ' > ' + str(dataTypes[ans_type]) + ' > ' + str(selected_year) + ' > ' + str(selected_month)) #SELECTED TO GENERATE df1 = df1.loc[str(selected_year) + '-' + str(selected_month)] ## TIME INTERVAL VERIFICATION df1 = df1.sort_index(ascending=True) # GET TIMES STRING max_time = df1.index.max() min_time = df1.index.min() ## FIND INDEX INTO ORIGINAL DATAFRAME idx_min = df.loc[df[0] == str(min_time)].index.values[0] idx_max = df.loc[df[0] == str(max_time)].index.values[0] ## LOC BETWEEN IDX locked_df_chk = df.loc[idx_min:idx_max] ## FINAL DF TO COMPARE final_df = locked_df_chk.copy() ## MOUNT INDEX OF DATES ACORDING TYPE OF DATA if dataTypes[ans_type] == 'MD' or dataTypes[ ans_type] == '10' or dataTypes[ans_type] == '25' or dataTypes[ ans_type] == '50': freqc = '10min' if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD': freqc = '1min' ## PASS COLUMN TO DATETIME locked_df_chk[0] = pd.to_datetime(locked_df_chk[0], format='%Y-%m-%d %H:%M:%S') ## MULTIINDEX mux = [] for i in range(len(head1)): mux.append([str(head1[i]).lower(), str(head2[i]).lower()]) mux = pd.MultiIndex.from_tuples(mux) ## GET ID OF STATION id_st = locked_df_chk[2].values[0] ## DETECT non-existent SENSOR non_sens_col = [] object_type_c = locked_df_chk.select_dtypes(include=['object']) for c in object_type_c: detc_mean = object_type_c[c].astype(float).mean() if str(detc_mean) == 'nan': non_sens_col.append(c) ## DESAPROVE WITH TIME INTERVAL ## DETECT TYPE OF FILE if dataTypes[ans_type] == 'MD' or dataTypes[ ans_type] == '10' or dataTypes[ans_type] == '25' or dataTypes[ ans_type] == '50': t_delta = pd.Timedelta(minutes=10) if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD': t_delta = pd.Timedelta(minutes=1) ## Generate all month days to compare year_month = locked_df_chk[0].dt.strftime('%Y-%m').values[0] # print(pd.Timestamp(year_month) + pd.offsets.MonthEnd(1) + pd.Timedelta(hours=24) - t_delta) month_generated = pd.date_range( start=pd.Timestamp(year_month), end=pd.Timestamp(year_month) + pd.offsets.MonthEnd(1) + pd.Timedelta(hours=24) - t_delta, # <-- 2018-08-31 with MonthEnd freq=freqc) ## CHECK DUPLICAT IN TIMESTAMP COLUMN times_dup = locked_df_chk[locked_df_chk.duplicated([0], keep=False)] group_tdup = times_dup.groupby(0) ## RESOLVE CHOKE TIME STAMP idx_first = [] for g, gdftum in group_tdup: # print(gdftum) idx_groups = gdftum.index for idxgg in range(len(idx_groups)): pass_idx = idx_groups[idxgg] - 1 if dataTypes[ans_type] == 'MD' or dataTypes[ ans_type] == '10' or dataTypes[ ans_type] == '25' or dataTypes[ans_type] == '50': t_delta = pd.Timedelta(minutes=10) locked_df_chk.loc[ idx_groups[idxgg], 0] = locked_df_chk.loc[pass_idx][0] + t_delta if dataTypes[ans_type] == 'SD' or dataTypes[ans_type] == 'TD': t_delta = pd.Timedelta(minutes=1) locked_df_chk.loc[ idx_groups[idxgg], 0] = locked_df_chk.loc[pass_idx][0] + t_delta ## CHECK DUPLICAT IN TIMESTAMP COLUMN locked_df_chk = locked_df_chk.drop_duplicates(subset=0) ## SET INDEX COLUMN OF TIME STAMP locked_df_chk = locked_df_chk.set_index(0) ## CHEACK DUPLICATED IN ALL COLUMNS locked_df_chk = locked_df_chk.drop_duplicates(keep='first') # FILL locked_df_chk = locked_df_chk.reindex(month_generated, fill_value=0) ## DESAPROVE WITH TIME INTERVAL check_t_interval = locked_df_chk check_t_interval = check_t_interval[min_time:max_time] # ## LIMITE TIME DETAL lim_delta = pd.Timedelta(minutes=50) totalDelta = t_delta for i, row in check_t_interval.iterrows(): if np.all(row[6:-1].values == 0): totalDelta = totalDelta + t_delta else: last_ro = i totalDelta = pd.Timedelta(minutes=0) if totalDelta >= lim_delta: print( 'Failed to generate file due to a longer time sequence of failures greater than ', lim_delta, '\n') print('') fail = check_t_interval[last_ro:i + t_delta] # fail[6:-1] = 3333 fail = fail.reset_index() fail.columns = mux print(fail) return None ## ADD NON-EXISTENT VALUES locked_df_chk[non_sens_col] = 5555 idx_values = locked_df_chk.loc[locked_df_chk[2] == 0].index ## ADD YEAR locked_df_chk.loc[idx_values, 3] = locked_df_chk.loc[idx_values].index.strftime( '%Y').values ## ADD JULIAN DAY locked_df_chk.loc[idx_values, 4] = locked_df_chk.loc[idx_values].index.strftime( '%j').values ## ADD MINUTE for hm in idx_values: s1 = hm.strftime('%d/%m/%Y 00:00') s2 = hm.strftime('%d/%m/%Y %H:%M') s1 = datetime.strptime(s1, '%d/%m/%Y %H:%M') s2 = datetime.strptime(s2, '%d/%m/%Y %H:%M') differenc = s2 - s1 minutes = divmod(differenc.seconds, 60) locked_df_chk.loc[hm, 5] = minutes[0] locked_df_chk[2] = id_st ## DATETIME COLUMNS + NON_SENSOR dt_non_se = [] dt_non_se.append(2) dt_non_se.append(3) dt_non_se.append(4) dt_non_se = dt_non_se + non_sens_col ### ADD VALUES 3333, diff_columns = np.setdiff1d(locked_df_chk.columns.values, dt_non_se) locked_df_chk.loc[idx_values, diff_columns] = 3333 # print(locked_df_chk['2020-09-09 13:30':'2020-09-09 17:20'].head(50)) ## SAVE if len(locked_df_chk) > 0: ## SAVE PROCESS output_dir = load_config()[0]['HISTORICAL_OUT'] + str( operational_stations[ans_file]).upper() + '/' + str( selected_year) + '/' output_file_name = str(operational_stations[ans_file]).upper( ) + '_' + str(selected_year) + '_' + str( locked_df_chk.index[0].strftime('%j')) + '_a_' + str( locked_df_chk.index[-1].strftime('%j')) + '_' + str( dataTypes[ans_type]) + '.dat' ## COUNT VALUES lost_counter = 0 nosen_counter = 0 for i, row in locked_df_chk.iterrows(): ## COUNTER FOR LOST FILES if 3333 in row.values[:]: lost_counter = lost_counter lost_counter += 1 ## COUNTER FOR LOST FILES if 5555 in row.values[:]: nosen_counter = nosen_counter nosen_counter += 1 ## RESET INDEX locked_df_chk = locked_df_chk.reset_index() # ADD MULTIINDEX locked_df_chk.columns = mux # CREATE DIR pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) ## CREATE VERSION DIRS pathlib.Path(output_dir + '/versions').mkdir(parents=True, exist_ok=True) file__ = output_dir + output_file_name # ## CHEACK IF FILES EXISTS if os.path.isfile(file__): warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) + ' \nLOST 3333 ROWS:-> ' + str(lost_counter) + '\nNO SENSOR 5555 ROWS:\n' + str(nosen_counter) + '\n') logging.warning(warningmsg) # print(warningmsg) # print(locked_df_chk.head()) ## CHECK LAST VERSION if len(os.listdir(output_dir + '/versions')) == 0: shutil.move( file__, output_dir + '/versions/' + output_file_name + '.v01') ## CREATE FILE locked_df_chk.to_csv(file__, index=False) warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) + ' File version(0) \nLOST 3333 ROWS:-> ' + str(lost_counter) + '\nNO SENSOR 5555 ROWS:\n' + str(nosen_counter) + '\n') logging.warning(warningmsg) else: versions = [ fn for fn in listdir(output_dir + '/versions/') if not fn.startswith('.') ] shutil.move( file__, output_dir + '/versions/' + output_file_name + 'v0' + str((int(versions[-1][-2:]) + 1))) warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) + ' File version(' + str(int(versions[-1][-2:]) + 1) + ') \nLOST 3333 ROWS:-> ' + str(lost_counter) + '\nNO SENSOR 5555 ROWS:\n' + str(nosen_counter) + '\n') logging.warning(warningmsg) print(warningmsg) ## CREATE FILE locked_df_chk.to_csv(file__, index=False) print(locked_df_chk) else: warningmsg = ('\nSTATION-> ' + str(output_file_name[:-4]) + ' File version(0) \nLOST 3333 ROWS:-> ' + str(lost_counter) + '\nNO SENSOR 5555 ROWS:\n' + str(nosen_counter) + '\n') logging.warning(warningmsg) ## CREATE FILE print(warningmsg) print(locked_df_chk) locked_df_chk.to_csv(file__, index=False) ## UPLOAD RESULTS ver_file_names = [ fn for fn in listdir(output_dir + str('/versions')) if not fn.startswith('.') ] if len(ver_file_names) > 0: last_file_version = sorted(ver_file_names)[-1] else: last_file_version = None file_to_upload = file__ print('\t\tUpload files to FTP: ') choice = input(""" (Y) - Yes (N) - No Please enter your choice: """) if choice == "Y" or choice == "y": connection(file_to_upload, operational_stations[ans_file], selected_year, output_file_name, last_file_version, operational_stations[ans_file]) elif choice == "N" or choice == "n": sys.exit elif choice == "Q" or choice == "q": sys.exit else: print("You must only select one option") print("Please try again") mainMenu()
def skycamera(): top_header('Main Menu > Sky Camera') print('\t\tPlease select an option') load_stations_02()
def download_files(directory, connection, station): top_header('Main Menu > Preprocessing Mode > Download Data > ' + str(station)) print('\t\tPlease select type of the data to download: ') choice = input(""" 0: All 1: Meteorological - MD 2: Solarimetric - SD 3: Anemometric - 10 4: Anemometric - 25 5: Anemometric - 50 B: Back Q: Quit Please enter your choice: """) if choice == "All" or choice == "0": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + str(station)).mkdir( parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + str(station)) print(config_file[0]['OPERATIONAL_IN'] + str(station)) connection.cwd(directory) for filename in connection.nlst('*.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "Meteorological" or choice == "1": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)).mkdir(parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)) connection.cwd(directory) for filename in connection.nlst('*MD.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "Solarimetric" or choice == "2": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)).mkdir(parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)) connection.cwd(directory) for filename in connection.nlst('*SD.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "Anemometric10" or choice == "3": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)).mkdir(parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)) connection.cwd(directory) for filename in connection.nlst('*10.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "Anemometric25" or choice == "4": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)).mkdir(parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)) connection.cwd(directory) for filename in connection.nlst('*25.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "Anemometric50" or choice == "5": pathlib.Path(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)).mkdir(parents=True, exist_ok=True) os.chdir(config_file[0]['OPERATIONAL_IN'] + '/' + str(station)) connection.cwd(directory) for filename in connection.nlst('*50.DAT'): fhandle = open(filename, 'wb') print('\t\tDownloading... ' + filename) connection.retrbinary('RETR ' + filename, fhandle.write) fhandle.close() print('All files has been downloaded!') elif choice == "b" or choice == "B": download_stations() elif choice == "Q" or choice == "q": sys.exit else: print("You must only select one option") print("Please try again") download_files()
def translate_historical(): operation_dir = load_config()[0]['HISTORICAL_OUT'] top_header('Main > Preprocessing > Translate Historical') print('\t\tPlease select one stations to translate historical data: ') historical_pats = [ fn for fn in listdir(operation_dir) if not fn.startswith('.') ] if len(historical_pats) == 0: print('There is no data to be formatted') input('Press Enter to return') ## SELECT STATION count = -1 for f in historical_pats: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file > count: print("\t\t Wrong selection.") continue selected_st = operation_dir + historical_pats[ans_file] + '/' top_header('Main > Preprocessing > Translate Historical > ' + str(historical_pats[ans_file])) print('\t\tPlease select one year file: ') select_year = [ fn for fn in listdir(selected_st) if not fn.startswith('.') ] ## SELECT FILE count = -1 for f in select_year: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_year = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_year > count: print("\t\t Wrong selection.") continue selected_year = selected_st + select_year[ans_year] + '/' top_header('Main > Preprocessing > Translate Historical > ' + str(historical_pats[ans_file]) + ' > ' + select_year[ans_year]) print('\t\tPlease select one file to translate file: ') select_files = [ fn for fn in listdir(selected_year) if not fn.startswith('.') and '.dat' in fn ] ## FOR FILE count = -1 for f in select_files: count = count + 1 print("\t\t [%s]" % count + f) while True: try: ans_file_ = int(input("\t\t Select Station: ")) except: print("\t\t Wrong selection") continue if ans_file_ > count: print("\t\t Wrong selection.") continue selected_file = selected_year + select_files[ans_file_] open_file(selected_file, historical_pats[ans_file], select_year[ans_year], select_files[ans_file_]) break break break