def process_knmi_dayvalues_selected(): '''Function asks for one or more wmo numbers to download their data''' console.header('START DOWNLOAD STATION(S) KNMI DATA DAY VALUES...', True) done, max = 0, (stations.list) while True: places = cask.ask_for_stations('\nSelect one or more stations ? ', stations.list) if utils.is_quit(places): break st = time.time_ns() for stat in places: daydata.process_data(stat) console.log(' ', True) console.log(vt.process_time('Total processing time is ', st), True) again = cask.ask_again(f'Do you want to download more stations ?', True) if utils.is_quit(again): break console.footer('END DOWNLOAD STATION(S) KNMI DATA DAY VALUES...', True)
def ask_for_yn( txt, default, space=False ): ok = False t = f'{txt}\n' t += '\t1) Yes\n' t += '\t2) No' while True: answ = ask_for_txt( t, default=default, space=space ) if not answ or utils.is_empthy(answ): # 'ie. y, n, no, yes 1, 2' answ = str(default) answ = answ.lower() if utils.is_quit(answ): return config.answer_quit[0] # Return first el for quit elif utils.is_yes(answ): return config.answer_yes[0] elif utils.is_no(answ): return config.answer_no[0] else: try: answi = int(answ) except ValueError: # Input was not a number pass else: # Input is a number if answi == 1: return config.answer_yes[0] elif answi == 2: return config.answer_no[0] console.log(f'Unknown option: {answ} ?\nTry again.', True)
def ask_for_a_month( txt, space=False ): l = [ '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', 'january', 'februari', 'march', 'april', 'mai', 'june', 'july', 'august', 'september', 'oktober', 'november', 'december', 'jan', 'feb', 'mar', 'apr', 'mai', 'jun', 'jul', 'aug', 'sep', 'okt', 'nov', 'dec' ] t = f'{txt}\n' t += vt.list_to_col( l, align='center', col=6, col_width='10' ) t += 'To select a month. Type in a month as above.' while True: answ = ask_for_txt( t, default=False, space=space ) if not answ or utils.is_empthy(answ): console.log('Please type in something ...', True) elif utils.is_quit(answ): res = config.answer_quit[0] # Return first el for quit break else: answ = answ.lower() if answ in l: res = vt.month_to_num( answ ) break else: ask( f'Month {answ} unknown !\nPress a key to try again...', True ) return res
def ask_for_a_day( txt, space=False ): days = [ 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 ] months = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ] l = list() for i in range( len(months) ): m = months[i] mm = f'0{m}' if m < 10 else f'{m}' for d in range(1, days[m-1]+1, 1): dd = f'0{d}' if d < 10 else f'{d}' l.append( f'{mm}{dd}' ) t = f'{txt}\n' t += vt.list_to_col( l, align='left', col=14, col_width=5 ) t += 'To select a day. Type in the day as above.' while True: answ = ask_for_txt( t, default=False, space=space ) if not answ or utils.is_empthy(answ): console.log('Please type in something ...', True) elif utils.is_quit(answ): res = config.answer_quit[0] # Return first el for quit break elif answ in l: res = answ break else: ask( f'Day {answ} unknown !\nPress a key to try again...', True ) return res
def table_allstats(): '''Function makes calculations for all statistics''' while True: console.header('START CALCULATE ALL STATISTICS...', True) # Ask for all in one ok, period, places, type, name = cask.ask_period_stations_type_name( 'all') if not ok: break console.header(f'CALCULATING ALL STATISTICS...', True) st = time.time_ns() path = allstats.calculate(places, period, name, type) console.log(vt.process_time('Total processing time is ', st), True) if type != 'cmd': fopen = cask.ask_to_open_with_app( f'\nOpen the file (type={type}) with your default application ?' ) if fopen: fio.open_with_app(path, 2, True) # Always ask for going back again = cask.ask_again( f'\nDo you want to make another all statistics table ?') if utils.is_quit(again): break console.footer(f'END CALCULATE ALL STATISTICS...', True)
def ask_for_date_with_check_data( stat, txt, space=False ): console.log(f'Loading data {stat.place} ...') ok, data = daydata.read( stat ) if not ok: input(f'Error reading data! {stat.place}') return config.answer_quit[0], np.array([]) sd = int( data[ 0, daydata.YYYYMMDD]) ed = int( data[-1, daydata.YYYYMMDD]) t = f'{txt}\n' t += f'Available range data for - {stat.place} - is from {sd} untill {ed}' while True: answ = ask_for_date(t, space) if utils.is_quit(answ): return config.answer_quit[0] # Return first el for quit, np.array([]) else: try: ymd = int(answ) except Exception as e: pass else: if ymd < sd or ymd > ed: console.log(f'Date {ymd} is out of range for {stat.place}') else: return answ, data
def s_to_bytes(s, charset, errors): try: b = s.encode(encoding=charset, errors=errors) except Exception as e: console.log(f'Fail convert to bytes with charset {charset}\nError {e}', True) else: return b return s
def read_stations_period( stations, per ): data = np.array([]) for station in stations: console.log(f'Read station {station.place}', True) ok, data_station = read_station_period ( station, per ) if ok: data = data_station if data.size == 0 else np.concatenate( (data, data_station) ) return data
def bytes_to_s(b, charset, errors): try: s = b.decode(encoding=charset, errors=errors) except Exception as e: console.log( f'Fail convert to string with charset {charset}.\nError:{e}', True) else: return s return b
def process( places, period, query ): # Read all data stations in a given period data = daydata.read_stations_period( places, period ) #= numpy array # Get all the days to search for console.log(f'Executing query: {query}', True) if query.find('and') == -1 and query.find('or') == -1: return query_simple( data, query ) # Process only one simple query else: return query_advanced( data, query ) # Process query with and, or
def download_and_read_file(url, file): ok, t = False, '' if has_internet(): ok = fio.download(url, file) if ok: ok, t = fio.read(file) else: console.log('No internet connection...', True) return ok, t
def process_knmi_dayvalues_all(): '''Function downloads, unzipped all knmi stations in the list''' console.header('START DOWNLOADING ALL DATA DAYVALUES KNMI STATIONS...', True) st = time.time_ns() for stat in stations.list: daydata.process_data(stat) console.log(vt.process_time('Total processing time is ', st), True) console.footer('END DOWNLOADING ALL STATIONS KNMI DATA DAY VALUES', True) cask.ask_back_to_main_menu()
def open_with_app(fname, verbose=False): '''Function opens a file''' try: # should work on Windows os.startfile(fname) except AttributeError: try: # Linux subprocess.call(['open', fname]) except Exception as e: console.log(vt.error('Open', e), verbose) else: console.log(vt.succes('Check'), verbose)
def ask(txt='?', space=False): t = clear(txt) if t: t = tr.txt( txt ) else: t = ' ... ? ' if space: console.log(' ', space) res = input(t) if space: console.log(' ', space) return res
def ask_for_txt(txt='?', default=False, space=False): t = f'{txt}\n' if default != False: t += vt.enter_default(default) + '\n' t += vt.enter_back_to_main() + '\n' t += ' ? ' if space: console.log(' ', space) answ = ask(t, space) if space: console.log(' ', space) return answ
def ask_for_query( txt, space=False ): info = False while True: t = f'{txt}\n' t += 'For example: TX > 35\n' if config.help_info or info: t += 'Possible properties are\n' t += "' gt', '> ' = greater than ie TG > 20 Warm nights\n" t += "' ge', '>=', ' ≥' = greater than and equal ie TX >= 30 Tropical days\n" t += "' lt', '< ' = less than ie TN < 0 Frosty days\n" t += "' le', '<=', ' ≤' = less than equal ie TX <= 0 Icy days\n" t += "' eq', '==' = equal ie DDVEC == 90 A day with a wind from the east\n" t += "' ne', '!=', '<>' = not equal ie RH != 0 A day with rain\n" t += "' or', '||' 'or ' ie SQ > 10 or TX >= 25 Sunny and warm days\n" t += "'and', '&&' 'and' ie RH > 10 and TX < 0 Most propably a day with snow\n" t += '\nPossible entities are\n' t += "'DDVEC' = Vector mean wind direction (degrees) 'FHVEC' = Vector mean windspeed (m/s)\n" t += "'FG' = Daily mean windspeed (in 0.1 m/s) 'FHX' = Maximum hourly mean windspeed (m/s)\n" t += "'FHN' = Minimum hourly mean windspeed (m/s) 'FXX' = Maximum wind gust (m/s)\n" t += "'TG' = Daily mean temperature in (°C) 'TN' = Minimum temperature (°C)\n" t += "'TX' = Maximum temperature (°C) 'T10N' = Minimum temperature at 10 cm (°C)\n" t += "'SQ' = Sunshine duration (hour) 'SP' = % of maximum sunshine duration\n" t += "'Q' = Global radiation (J/cm2) 'DR' = Precipitation duration (hour)\n" t += "'RH' = Daily precipitation amount (mm) 'RHX' = Maximum hourly precipitation (mm)\n" t += "'PG' = Daily mean sea level pressure (hPa) 'PX' = Maximum hourly sea level pressure (hPa)\n" t += "'PN' = Minimum hourly sea level pressure (hPa) 'EV24' = Potential evapotranspiration (mm)\n" t += "'VVN' = Minimum visibility 0: <100m, 1:100-200m, 2:200-300m,..., 49:4900-5000m, 50:5-6 km, \n" t += ' 56:6-7km, 57:7-8km,..., 79:29-30km, 80:30-35km, 81:35-40km,..., 89: >70km)\n' t += "'VVX' = Maximum visibility 0: <100 m, 1:100-200 m, 2:200-300 m,..., 49:4900-5000 m, 50:5-6 km, \n" t += ' 56:6-7km, 57:7-8km,..., 79:29-30km, 80:30-35km, 81:35-40km,..., 89: >70km)\n' t += "'NG' = Mean daily cloud cover (octants) 'UG' = Daily mean relative atmospheric humidity (%)\n" t += "'UX' = Maximum atmospheric humidity (%) 'UN' = Minimum relative atmospheric humidity (%)" else: t += "Type 'i' for more info..." answ = ask_for_txt( t, default=False, space=space ) # TODO MAKE ADVANCED CHECKS if not answ or utils.is_empthy(answ): console.log('Please type in something ...', True) elif answ == 'i': info = True elif utils.is_quit(answ): return config.answer_quit[0] # Return first el for quit # TODO # elif validate.query(answ): # TODO # return answ else: return utils.make_query_txt_only( answ ) return answ # No checking for now
def is_float(val): ok = True try: if val is None: ok = False elif is_nan(val): ok = False else: f = float(val) except Exception as e: console.log(f'Digit {val} is no float.') ok = False return ok
def search_for_days(): '''Function searches files for days with specific values. ie > 30 degrees''' while True: console.header('START SEARCHING FOR SPECIFIC DAYS...', True) period = cask.ask_for_period( '\nFor which time periode do you want to search ? ') if utils.is_quit(period): break places = cask.ask_for_stations( '\nSelect one or more weather stations ?') if not places: cask.ask_back_to_main_menu() break elif utils.is_quit(places): break query = cask.ask_for_query('\nType in a query for selecting days ? ') if utils.is_quit(query): break type = cask.ask_for_file_type('\nSelect output filetype ? ', cfg.default_output) if utils.is_quit(type): break fname = f'days-{utils.make_query_txt_only(query)}-{period}-{utils.now_for_file()}' fname = fname.replace('*', 'x').replace(' ', '') fname = cask.ask_for_file_name( '\nGive a name for the file ? <optional>', fname) if utils.is_quit(fname): break st = time.time_ns() path = search4days.calculate(places, period, query, type, fname) console.log(vt.process_time('Total processing time is ', st), True) if type in ['text', 'html']: fopen = cask.ask_to_open_with_app( f'\nOpen the file (type={type}) with your default application ?' ) if fopen: fio.open_with_app(path, 2, True) # Always ask for going back again = cask.ask_again(f'Do you want to search for days again ?', True) if utils.is_quit(again): break console.footer('END SEARCH FOR DAYS...', True)
def ask_for_date( txt, space=False): while True: answ = ask_for_txt( txt, default=False, space=space ) if not answ or utils.is_empthy(answ): console.log('Please type in something ...', True) continue elif utils.is_quit(answ): answ = config.answer_quit[0] # Return first el for quit break elif validate.yyyymmdd(answ): break else: console.log(f'Error in date: {answ}\n', True) return answ
def ask_for_period( txt='', space=False ): info = False while True: t= f'{txt} \n' t += 'Period start - end \n' t += 'Default format yyyymmdd-yyyymmdd \n' t += 'For example: 20200510-20200520 \n' if config.help_info or info: t += 'Formats with a wild card * \n' t += ' --- still in development --- \n' t += ' ******** selects all the available data (=8x*)\n' t += ' **** selects (current) year (=4x*)\n' t += ' ** selects (current) month (=2x*)\n' t += ' yyyy**** selects the whole year yyyy\n' t += ' yyyymm** selects the month mm in the year yyyy\n' t += ' ****mm** selects a month mm for every year \n' t += ' ****mmdd selects a monthday mmdd for every year \n' t += ' yyyy****-yyyy**** selects a full year from yyyy untill yyyy \n' t += ' yyyymmdd-yyyy*mmdd selects a day mmdd in a year from start day to endyear\n' t += ' yyyymmdd-yyyy*mmdd* selects a certain period from mmdd=mmdd* in a year from yyyy to yyyy\n' t += 'Examples wildcard * \n' t += ' 2015**** selects the year 2015 \n' t += ' 201101**-202001** selects all januarys from 2011 unto 2020 \n' t += ' ****07** selects all julys from avalaible data \n' t += ' ****1225 selects all 25 decembers from avalaible data' else: t+= "Type 'i' for more info..." answ = ask_for_txt( t, default=False, space=space ) if not answ or utils.is_empthy(answ): console.log('\nPlease type in something...\n', True) elif utils.is_quit(answ): answ = config.answer_quit[0] # Return first el for quit break elif answ == 'i': info = True else: if daydata.check_periode( answ ): # TODO answ = answ.replace(' ', '') break else: err = f'Period of format {answ} is unknown... Press a key... ' ask(err) return answ
def month_extremes(): '''Function calculates extremes in a month in''' while True: console.header('START CALCULATE EXTREMES IN A MONTH...', True) month = cask.ask_for_a_month( '\nFor which month do you want to calculate the extremes ? ') if utils.is_quit(month): break period = cask.ask_for_period( f'\nFor what time periode do you want to caculate the extremes in the month {month} ? ' ) if utils.is_quit(period): break places = cask.ask_for_stations( '\nSelect one or more weather stations ? ') if utils.is_quit(places): break type = cask.ask_for_file_type('\nSelect output filetype ? ', cfg.default_output) if utils.is_quit(type): break fname = f'monthextremes-{month}-{period}-{utils.now_for_file()}' fname = fname.replace('*', 'x').replace(' ', '') fname = cask.ask_for_file_name( '\nGive a name for the file ? <optional>', fname) if utils.is_quit(fname): break st = time.time_ns() path = mmonthextremes.calculate(places, period, month, type, fname) console.log(vt.process_time('Total processing time is ', st), True) if type in ['text', 'html']: fopen = cask.ask_to_open_with_app( f'\nOpen the file (type={type}) with your default application ?' ) if fopen: fio.open_with_app(path, 2, True) # Always ask for going back again = cask.ask_again( f'Do you want to calculate extremes for a month again ?', True) if utils.is_quit(again): break console.footer('END CALCULATE EXTREMES IN A MONTH...', True)
def terms_days(data, entity, operator, value): '''Function select days based on terms like TX > 30 for example''' ndx = daydata.ndx_ent(entity) # Get index for entity in data matrix op = operator.lower() # Make operator lowercase f = df(value, entity) # Make input value equal to data in matrix if is_operator(op): # Check for allowed operator if op in ['gt', '>']: sel = np.where(data[:, ndx] > f) elif op in ['ge', '>=', '≥']: sel = np.where(data[:, ndx] >= f) elif op in ['eq', '==']: sel = np.where(data[:, ndx] == f) elif op in ['lt', '<']: sel = np.where(data[:, ndx] < f) elif op in ['le', '<=', '≤']: sel = np.where(data[:, ndx] <= f) elif op in ['ne', '!=', '<>']: sel = np.where(data[:, ndx] != f) else: console.log(f'Error in operator {op} in terms for day...') else: # Wrong input console.log(f'Error. Operator {op} is unknown...') return data[sel] # Return all days where the selected terms are true
def day( station, yyyymmdd ): '''Function return a list of the dayvalues''' ok, data = False, np.array([]) if validate.yyyymmdd(yyyymmdd): # Validate date ok, data = read(station) if ok: # Date is read fine ndx = ndx_ent('YYYYMMDD') ymd, s_ymd, e_ymd = int(yyyymmdd), data[0,ndx], data[-1,ndx] # Check ranges and correct anyway if ymd < s_ymd: ymd = s_ymd console.log(f'Date {s_ymd} out range of data. First available date {ymd} used.', True) elif ymd > s_ymd: ymd = e_ymd console.log(f'Date {e_ymd} out range of data. First available date {ymd} used.', True) data = data[np.where(data[:,ndx] == ymd)] # Get values correct date return ok, data[0]
def download(url, file, verbose=False): ok = False console.log(f'Download: {url}', verbose) console.log(f'To: {file}', verbose) try: with threading.Lock(): urllib.request.urlretrieve(url, file) except Exception as e: console.log(vt.error('Download', e), verbose) else: console.log(vt.succes('Download'), verbose) ok = True return ok
def year_extremes(): '''Function calculates extremes in a year''' while True: console.header('START CALCULATE EXTREMES IN A YEAR...', True) year = cask.ask_for_a_year('\nSelect a year for the extremes ? ') if utils.is_quit(year): break places = cask.ask_for_stations( '\nSelect one or more weather stations ? ') if utils.is_quit(places): break type = cask.ask_for_file_type('\nSelect output filetype ? ', cfg.default_output) if utils.is_quit(type): break fname = f'yearextremes-{year}-{utils.now_for_file()}' fname = fname.replace('*', 'x').replace(' ', '') fname = cask.ask_for_file_name( '\nGive a name for the file ? <optional>', fname) if utils.is_quit(fname): break st = time.time_ns() path = myearextremes.calculate(places, year, type, fname) console.log(vt.process_time('Total processing time is ', st), True) if type in ['text', 'html']: fopen = cask.ask_to_open_with_app( f'\nOpen the file (type={type}) with your default application ?' ) if fopen: fio.open_with_app(path) # Always ask for going back again = cask.ask_again( f'Do you want to calculate extremes for a year again ?', True) if utils.is_quit(again): break console.footer('END CALCULATE EXTREMES IN A YEAR...', True)
def ask_for_one_station( txt, l_map=False, space=False): result = False if l_map == False: l_map = utils.only_existing_stations_in_map() # Update l if len(l_map) != 0: t = f'{txt}\n' t += vt.list_to_col(l, 'left', 3, 25) t += 'To add a station, give a wmo-number or a city name' while True: answ = ask_for_txt( t, default=False, space=space ) if utils.is_empthy(answ): console.log('Please type in something ...', True) elif utils.is_quit(answ): result = config.answer_quit[0] # Return first el for quit break elif stations.name_in_list(answ, l_map) or \ stations.wmo_in_list(answ, l_map): result = stations.find_by_wmo_or_name(answ, l_map) console.log(f'{result.wmo}: {result.place} {result.province} selected', True) break else: ask( f'Station: {answ} unknown !\nPress a key to try again...', True ) else: t = 'No weatherdata found in data map. Download weatherdata first.' console.log(t, True) return result
def ask_type_options(txt, l, default=config.default_output, space=False): t = f'{txt}\n' i, max = 1, len(l) while True: t += f'\t{i}) {l[i-1]}' if i < max: i, t = i + 1, t + '\n' else: break while True: answ = ask_for_txt( t, default=default, space=space ) if not answ or utils.is_empthy(answ): i = 0 while i < max: if l[i] == default: return default elif str(i) == str(default): return l[i] i += 1 if utils.is_quit(answ): return config.answer_quit[0] # Return first el for quit else: try: answi = int(answ) except ValueError: # Input is not a number if answ in l: return answ else: # Input is a number i = 0 while i < max: if i + 1 == answi: return l[i] i += 1 console.log(f'Unknown option: {answ} ?\nTry again.', True)
def unzip(zip, txt, verbose=False): ok = False console.log(f'Unzip: {zip}', verbose) console.log(f'To: {txt}', verbose) # TODO force to txt file try: with threading.Lock(): dir_txt = os.path.dirname(txt) with ZipFile(zip, 'r') as z: z.extractall(dir_txt) except Exception as e: console.log(vt.error('Unzip', e), verbose) else: console.log(vt.succes('Unzip'), verbose) ok = True return ok
def delete(fname, verbose=False): '''Function deletes a file if exists''' ok = False console.log(f'Delete: {fname}', verbose) if check(fname): try: Path(fname).unlink() # Remove file except Exception as e: console.log(vt.error('Delete', e), verbose) else: console.log(vt.succes('Delete'), verbose) ok = True else: console.log(f'Delete cannot. File does not exist', verbose) return ok
def mk_dir(path, verbose=False): '''Function makes a map if not already exists''' ok = False console.log(f'Make dir: {path}', verbose) try: if os.path.isdir(path): console.log('Map not made because it already exists.', verbose) return True else: os.makedirs(path) except Exception as e: console.log(vt.error('Make directory', e), verbose) else: console.log(vt.succes('Make directory'), verbose) ok = True return ok