def process_knmi_dayvalues_selected():
'''Function asks for one or more wmo numbers to download their data'''
console.header('START DOWNLOAD STATION(S) KNMI DATA DAY VALUES...', True)
done, max = 0, (stations.list)
while True:
places = cask.ask_for_stations('\nSelect one or more stations ? ',
stations.list)
if utils.is_quit(places):
break
st = time.time_ns()
for stat in places:
daydata.process_data(stat)
console.log(' ', True)
console.log(vt.process_time('Total processing time is ', st), True)
again = cask.ask_again(f'Do you want to download more stations ?',
True)
if utils.is_quit(again):
break
console.footer('END DOWNLOAD STATION(S) KNMI DATA DAY VALUES...', True)
def ask_for_yn( txt, default, space=False ):
ok = False
t = f'{txt}\n'
t += '\t1) Yes\n'
t += '\t2) No'
while True:
answ = ask_for_txt( t, default=default, space=space )
if not answ or utils.is_empthy(answ): # 'ie. y, n, no, yes 1, 2'
answ = str(default)
answ = answ.lower()
if utils.is_quit(answ):
return config.answer_quit[0] # Return first el for quit
elif utils.is_yes(answ):
return config.answer_yes[0]
elif utils.is_no(answ):
return config.answer_no[0]
else:
try:
answi = int(answ)
except ValueError: # Input was not a number
pass
else: # Input is a number
if answi == 1:
return config.answer_yes[0]
elif answi == 2:
return config.answer_no[0]
console.log(f'Unknown option: {answ} ?\nTry again.', True)
def ask_for_a_month( txt, space=False ):
l = [ '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12',
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12',
'january', 'februari', 'march', 'april', 'mai', 'june', 'july',
'august', 'september', 'oktober', 'november', 'december',
'jan', 'feb', 'mar', 'apr', 'mai', 'jun', 'jul',
'aug', 'sep', 'okt', 'nov', 'dec' ]
t = f'{txt}\n'
t += vt.list_to_col( l, align='center', col=6, col_width='10' )
t += 'To select a month. Type in a month as above.'
while True:
answ = ask_for_txt( t, default=False, space=space )
if not answ or utils.is_empthy(answ):
console.log('Please type in something ...', True)
elif utils.is_quit(answ):
res = config.answer_quit[0] # Return first el for quit
break
else:
answ = answ.lower()
if answ in l:
res = vt.month_to_num( answ )
break
else:
ask( f'Month {answ} unknown !\nPress a key to try again...', True )
return res
def ask_for_a_day( txt, space=False ):
days = [ 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 ]
months = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ]
l = list()
for i in range( len(months) ):
m = months[i]
mm = f'0{m}' if m < 10 else f'{m}'
for d in range(1, days[m-1]+1, 1):
dd = f'0{d}' if d < 10 else f'{d}'
l.append( f'{mm}{dd}' )
t = f'{txt}\n'
t += vt.list_to_col( l, align='left', col=14, col_width=5 )
t += 'To select a day. Type in the day as above.'
while True:
answ = ask_for_txt( t, default=False, space=space )
if not answ or utils.is_empthy(answ):
console.log('Please type in something ...', True)
elif utils.is_quit(answ):
res = config.answer_quit[0] # Return first el for quit
break
elif answ in l:
res = answ
break
else:
ask( f'Day {answ} unknown !\nPress a key to try again...', True )
return res
def table_allstats():
'''Function makes calculations for all statistics'''
while True:
console.header('START CALCULATE ALL STATISTICS...', True)
# Ask for all in one
ok, period, places, type, name = cask.ask_period_stations_type_name(
'all')
if not ok: break
console.header(f'CALCULATING ALL STATISTICS...', True)
st = time.time_ns()
path = allstats.calculate(places, period, name, type)
console.log(vt.process_time('Total processing time is ', st), True)
if type != 'cmd':
fopen = cask.ask_to_open_with_app(
f'\nOpen the file (type={type}) with your default application ?'
)
if fopen: fio.open_with_app(path, 2, True)
# Always ask for going back
again = cask.ask_again(
f'\nDo you want to make another all statistics table ?')
if utils.is_quit(again):
break
console.footer(f'END CALCULATE ALL STATISTICS...', True)
def ask_for_date_with_check_data( stat, txt, space=False ):
console.log(f'Loading data {stat.place} ...')
ok, data = daydata.read( stat )
if not ok:
input(f'Error reading data! {stat.place}')
return config.answer_quit[0], np.array([])
sd = int( data[ 0, daydata.YYYYMMDD])
ed = int( data[-1, daydata.YYYYMMDD])
t = f'{txt}\n'
t += f'Available range data for - {stat.place} - is from {sd} untill {ed}'
while True:
answ = ask_for_date(t, space)
if utils.is_quit(answ):
return config.answer_quit[0] # Return first el for quit, np.array([])
else:
try:
ymd = int(answ)
except Exception as e:
pass
else:
if ymd < sd or ymd > ed:
console.log(f'Date {ymd} is out of range for {stat.place}')
else:
return answ, data
def s_to_bytes(s, charset, errors):
try:
b = s.encode(encoding=charset, errors=errors)
except Exception as e:
console.log(f'Fail convert to bytes with charset {charset}\nError {e}',
True)
else:
return b
return s
def read_stations_period( stations, per ):
data = np.array([])
for station in stations:
console.log(f'Read station {station.place}', True)
ok, data_station = read_station_period ( station, per )
if ok:
data = data_station if data.size == 0 else np.concatenate( (data, data_station) )
return data
def bytes_to_s(b, charset, errors):
try:
s = b.decode(encoding=charset, errors=errors)
except Exception as e:
console.log(
f'Fail convert to string with charset {charset}.\nError:{e}', True)
else:
return s
return b
def process( places, period, query ):
# Read all data stations in a given period
data = daydata.read_stations_period( places, period ) #= numpy array
# Get all the days to search for
console.log(f'Executing query: {query}', True)
if query.find('and') == -1 and query.find('or') == -1:
return query_simple( data, query ) # Process only one simple query
else:
return query_advanced( data, query ) # Process query with and, or
def download_and_read_file(url, file):
ok, t = False, ''
if has_internet():
ok = fio.download(url, file)
if ok:
ok, t = fio.read(file)
else:
console.log('No internet connection...', True)
return ok, t
def process_knmi_dayvalues_all():
'''Function downloads, unzipped all knmi stations in the list'''
console.header('START DOWNLOADING ALL DATA DAYVALUES KNMI STATIONS...',
True)
st = time.time_ns()
for stat in stations.list:
daydata.process_data(stat)
console.log(vt.process_time('Total processing time is ', st), True)
console.footer('END DOWNLOADING ALL STATIONS KNMI DATA DAY VALUES', True)
cask.ask_back_to_main_menu()
def open_with_app(fname, verbose=False):
'''Function opens a file'''
try: # should work on Windows
os.startfile(fname)
except AttributeError:
try: # Linux
subprocess.call(['open', fname])
except Exception as e:
console.log(vt.error('Open', e), verbose)
else:
console.log(vt.succes('Check'), verbose)
def ask(txt='?', space=False):
t = clear(txt)
if t:
t = tr.txt( txt )
else:
t = ' ... ? '
if space: console.log(' ', space)
res = input(t)
if space: console.log(' ', space)
return res
def ask_for_txt(txt='?', default=False, space=False):
t = f'{txt}\n'
if default != False:
t += vt.enter_default(default) + '\n'
t += vt.enter_back_to_main() + '\n'
t += ' ? '
if space: console.log(' ', space)
answ = ask(t, space)
if space: console.log(' ', space)
return answ
def ask_for_query( txt, space=False ):
info = False
while True:
t = f'{txt}\n'
t += 'For example: TX > 35\n'
if config.help_info or info:
t += 'Possible properties are\n'
t += "' gt', '> ' = greater than ie TG > 20 Warm nights\n"
t += "' ge', '>=', ' ≥' = greater than and equal ie TX >= 30 Tropical days\n"
t += "' lt', '< ' = less than ie TN < 0 Frosty days\n"
t += "' le', '<=', ' ≤' = less than equal ie TX <= 0 Icy days\n"
t += "' eq', '==' = equal ie DDVEC == 90 A day with a wind from the east\n"
t += "' ne', '!=', '<>' = not equal ie RH != 0 A day with rain\n"
t += "' or', '||' 'or ' ie SQ > 10 or TX >= 25 Sunny and warm days\n"
t += "'and', '&&' 'and' ie RH > 10 and TX < 0 Most propably a day with snow\n"
t += '\nPossible entities are\n'
t += "'DDVEC' = Vector mean wind direction (degrees) 'FHVEC' = Vector mean windspeed (m/s)\n"
t += "'FG' = Daily mean windspeed (in 0.1 m/s) 'FHX' = Maximum hourly mean windspeed (m/s)\n"
t += "'FHN' = Minimum hourly mean windspeed (m/s) 'FXX' = Maximum wind gust (m/s)\n"
t += "'TG' = Daily mean temperature in (°C) 'TN' = Minimum temperature (°C)\n"
t += "'TX' = Maximum temperature (°C) 'T10N' = Minimum temperature at 10 cm (°C)\n"
t += "'SQ' = Sunshine duration (hour) 'SP' = % of maximum sunshine duration\n"
t += "'Q' = Global radiation (J/cm2) 'DR' = Precipitation duration (hour)\n"
t += "'RH' = Daily precipitation amount (mm) 'RHX' = Maximum hourly precipitation (mm)\n"
t += "'PG' = Daily mean sea level pressure (hPa) 'PX' = Maximum hourly sea level pressure (hPa)\n"
t += "'PN' = Minimum hourly sea level pressure (hPa) 'EV24' = Potential evapotranspiration (mm)\n"
t += "'VVN' = Minimum visibility 0: <100m, 1:100-200m, 2:200-300m,..., 49:4900-5000m, 50:5-6 km, \n"
t += ' 56:6-7km, 57:7-8km,..., 79:29-30km, 80:30-35km, 81:35-40km,..., 89: >70km)\n'
t += "'VVX' = Maximum visibility 0: <100 m, 1:100-200 m, 2:200-300 m,..., 49:4900-5000 m, 50:5-6 km, \n"
t += ' 56:6-7km, 57:7-8km,..., 79:29-30km, 80:30-35km, 81:35-40km,..., 89: >70km)\n'
t += "'NG' = Mean daily cloud cover (octants) 'UG' = Daily mean relative atmospheric humidity (%)\n"
t += "'UX' = Maximum atmospheric humidity (%) 'UN' = Minimum relative atmospheric humidity (%)"
else:
t += "Type 'i' for more info..."
answ = ask_for_txt( t, default=False, space=space )
# TODO MAKE ADVANCED CHECKS
if not answ or utils.is_empthy(answ):
console.log('Please type in something ...', True)
elif answ == 'i':
info = True
elif utils.is_quit(answ):
return config.answer_quit[0] # Return first el for quit
# TODO
# elif validate.query(answ): # TODO
# return answ
else:
return utils.make_query_txt_only( answ )
return answ # No checking for now
def is_float(val):
ok = True
try:
if val is None:
ok = False
elif is_nan(val):
ok = False
else:
f = float(val)
except Exception as e:
console.log(f'Digit {val} is no float.')
ok = False
return ok
def search_for_days():
'''Function searches files for days with specific values. ie > 30 degrees'''
while True:
console.header('START SEARCHING FOR SPECIFIC DAYS...', True)
period = cask.ask_for_period(
'\nFor which time periode do you want to search ? ')
if utils.is_quit(period): break
places = cask.ask_for_stations(
'\nSelect one or more weather stations ?')
if not places:
cask.ask_back_to_main_menu()
break
elif utils.is_quit(places):
break
query = cask.ask_for_query('\nType in a query for selecting days ? ')
if utils.is_quit(query):
break
type = cask.ask_for_file_type('\nSelect output filetype ? ',
cfg.default_output)
if utils.is_quit(type):
break
fname = f'days-{utils.make_query_txt_only(query)}-{period}-{utils.now_for_file()}'
fname = fname.replace('*', 'x').replace(' ', '')
fname = cask.ask_for_file_name(
'\nGive a name for the file ? <optional>', fname)
if utils.is_quit(fname):
break
st = time.time_ns()
path = search4days.calculate(places, period, query, type, fname)
console.log(vt.process_time('Total processing time is ', st), True)
if type in ['text', 'html']:
fopen = cask.ask_to_open_with_app(
f'\nOpen the file (type={type}) with your default application ?'
)
if fopen:
fio.open_with_app(path, 2, True)
# Always ask for going back
again = cask.ask_again(f'Do you want to search for days again ?', True)
if utils.is_quit(again):
break
console.footer('END SEARCH FOR DAYS...', True)
def ask_for_date( txt, space=False):
while True:
answ = ask_for_txt( txt, default=False, space=space )
if not answ or utils.is_empthy(answ):
console.log('Please type in something ...', True)
continue
elif utils.is_quit(answ):
answ = config.answer_quit[0] # Return first el for quit
break
elif validate.yyyymmdd(answ):
break
else:
console.log(f'Error in date: {answ}\n', True)
return answ
def ask_for_period( txt='', space=False ):
info = False
while True:
t= f'{txt} \n'
t += 'Period start - end \n'
t += 'Default format yyyymmdd-yyyymmdd \n'
t += 'For example: 20200510-20200520 \n'
if config.help_info or info:
t += 'Formats with a wild card * \n'
t += ' --- still in development --- \n'
t += ' ******** selects all the available data (=8x*)\n'
t += ' **** selects (current) year (=4x*)\n'
t += ' ** selects (current) month (=2x*)\n'
t += ' yyyy**** selects the whole year yyyy\n'
t += ' yyyymm** selects the month mm in the year yyyy\n'
t += ' ****mm** selects a month mm for every year \n'
t += ' ****mmdd selects a monthday mmdd for every year \n'
t += ' yyyy****-yyyy**** selects a full year from yyyy untill yyyy \n'
t += ' yyyymmdd-yyyy*mmdd selects a day mmdd in a year from start day to endyear\n'
t += ' yyyymmdd-yyyy*mmdd* selects a certain period from mmdd=mmdd* in a year from yyyy to yyyy\n'
t += 'Examples wildcard * \n'
t += ' 2015**** selects the year 2015 \n'
t += ' 201101**-202001** selects all januarys from 2011 unto 2020 \n'
t += ' ****07** selects all julys from avalaible data \n'
t += ' ****1225 selects all 25 decembers from avalaible data'
else:
t+= "Type 'i' for more info..."
answ = ask_for_txt( t, default=False, space=space )
if not answ or utils.is_empthy(answ):
console.log('\nPlease type in something...\n', True)
elif utils.is_quit(answ):
answ = config.answer_quit[0] # Return first el for quit
break
elif answ == 'i':
info = True
else:
if daydata.check_periode( answ ): # TODO
answ = answ.replace(' ', '')
break
else:
err = f'Period of format {answ} is unknown... Press a key... '
ask(err)
return answ
def month_extremes():
'''Function calculates extremes in a month in'''
while True:
console.header('START CALCULATE EXTREMES IN A MONTH...', True)
month = cask.ask_for_a_month(
'\nFor which month do you want to calculate the extremes ? ')
if utils.is_quit(month): break
period = cask.ask_for_period(
f'\nFor what time periode do you want to caculate the extremes in the month {month} ? '
)
if utils.is_quit(period): break
places = cask.ask_for_stations(
'\nSelect one or more weather stations ? ')
if utils.is_quit(places): break
type = cask.ask_for_file_type('\nSelect output filetype ? ',
cfg.default_output)
if utils.is_quit(type): break
fname = f'monthextremes-{month}-{period}-{utils.now_for_file()}'
fname = fname.replace('*', 'x').replace(' ', '')
fname = cask.ask_for_file_name(
'\nGive a name for the file ? <optional>', fname)
if utils.is_quit(fname): break
st = time.time_ns()
path = mmonthextremes.calculate(places, period, month, type, fname)
console.log(vt.process_time('Total processing time is ', st), True)
if type in ['text', 'html']:
fopen = cask.ask_to_open_with_app(
f'\nOpen the file (type={type}) with your default application ?'
)
if fopen:
fio.open_with_app(path, 2, True)
# Always ask for going back
again = cask.ask_again(
f'Do you want to calculate extremes for a month again ?', True)
if utils.is_quit(again):
break
console.footer('END CALCULATE EXTREMES IN A MONTH...', True)
def terms_days(data, entity, operator, value):
'''Function select days based on terms like TX > 30 for example'''
ndx = daydata.ndx_ent(entity) # Get index for entity in data matrix
op = operator.lower() # Make operator lowercase
f = df(value, entity) # Make input value equal to data in matrix
if is_operator(op): # Check for allowed operator
if op in ['gt', '>']: sel = np.where(data[:, ndx] > f)
elif op in ['ge', '>=', '≥']: sel = np.where(data[:, ndx] >= f)
elif op in ['eq', '==']: sel = np.where(data[:, ndx] == f)
elif op in ['lt', '<']: sel = np.where(data[:, ndx] < f)
elif op in ['le', '<=', '≤']: sel = np.where(data[:, ndx] <= f)
elif op in ['ne', '!=', '<>']: sel = np.where(data[:, ndx] != f)
else:
console.log(f'Error in operator {op} in terms for day...')
else: # Wrong input
console.log(f'Error. Operator {op} is unknown...')
return data[sel] # Return all days where the selected terms are true
def day( station, yyyymmdd ):
'''Function return a list of the dayvalues'''
ok, data = False, np.array([])
if validate.yyyymmdd(yyyymmdd): # Validate date
ok, data = read(station)
if ok: # Date is read fine
ndx = ndx_ent('YYYYMMDD')
ymd, s_ymd, e_ymd = int(yyyymmdd), data[0,ndx], data[-1,ndx]
# Check ranges and correct anyway
if ymd < s_ymd:
ymd = s_ymd
console.log(f'Date {s_ymd} out range of data. First available date {ymd} used.', True)
elif ymd > s_ymd:
ymd = e_ymd
console.log(f'Date {e_ymd} out range of data. First available date {ymd} used.', True)
data = data[np.where(data[:,ndx] == ymd)] # Get values correct date
return ok, data[0]
def download(url, file, verbose=False):
ok = False
console.log(f'Download: {url}', verbose)
console.log(f'To: {file}', verbose)
try:
with threading.Lock():
urllib.request.urlretrieve(url, file)
except Exception as e:
console.log(vt.error('Download', e), verbose)
else:
console.log(vt.succes('Download'), verbose)
ok = True
return ok
def year_extremes():
'''Function calculates extremes in a year'''
while True:
console.header('START CALCULATE EXTREMES IN A YEAR...', True)
year = cask.ask_for_a_year('\nSelect a year for the extremes ? ')
if utils.is_quit(year): break
places = cask.ask_for_stations(
'\nSelect one or more weather stations ? ')
if utils.is_quit(places): break
type = cask.ask_for_file_type('\nSelect output filetype ? ',
cfg.default_output)
if utils.is_quit(type): break
fname = f'yearextremes-{year}-{utils.now_for_file()}'
fname = fname.replace('*', 'x').replace(' ', '')
fname = cask.ask_for_file_name(
'\nGive a name for the file ? <optional>', fname)
if utils.is_quit(fname): break
st = time.time_ns()
path = myearextremes.calculate(places, year, type, fname)
console.log(vt.process_time('Total processing time is ', st), True)
if type in ['text', 'html']:
fopen = cask.ask_to_open_with_app(
f'\nOpen the file (type={type}) with your default application ?'
)
if fopen:
fio.open_with_app(path)
# Always ask for going back
again = cask.ask_again(
f'Do you want to calculate extremes for a year again ?', True)
if utils.is_quit(again):
break
console.footer('END CALCULATE EXTREMES IN A YEAR...', True)
def ask_for_one_station( txt, l_map=False, space=False):
result = False
if l_map == False:
l_map = utils.only_existing_stations_in_map() # Update l
if len(l_map) != 0:
t = f'{txt}\n'
t += vt.list_to_col(l, 'left', 3, 25)
t += 'To add a station, give a wmo-number or a city name'
while True:
answ = ask_for_txt( t, default=False, space=space )
if utils.is_empthy(answ):
console.log('Please type in something ...', True)
elif utils.is_quit(answ):
result = config.answer_quit[0] # Return first el for quit
break
elif stations.name_in_list(answ, l_map) or \
stations.wmo_in_list(answ, l_map):
result = stations.find_by_wmo_or_name(answ, l_map)
console.log(f'{result.wmo}: {result.place} {result.province} selected', True)
break
else:
ask( f'Station: {answ} unknown !\nPress a key to try again...', True )
else:
t = 'No weatherdata found in data map. Download weatherdata first.'
console.log(t, True)
return result
def ask_type_options(txt, l, default=config.default_output, space=False):
t = f'{txt}\n'
i, max = 1, len(l)
while True:
t += f'\t{i}) {l[i-1]}'
if i < max:
i, t = i + 1, t + '\n'
else:
break
while True:
answ = ask_for_txt( t, default=default, space=space )
if not answ or utils.is_empthy(answ):
i = 0
while i < max:
if l[i] == default:
return default
elif str(i) == str(default):
return l[i]
i += 1
if utils.is_quit(answ):
return config.answer_quit[0] # Return first el for quit
else:
try:
answi = int(answ)
except ValueError: # Input is not a number
if answ in l:
return answ
else: # Input is a number
i = 0
while i < max:
if i + 1 == answi:
return l[i]
i += 1
console.log(f'Unknown option: {answ} ?\nTry again.', True)
def unzip(zip, txt, verbose=False):
ok = False
console.log(f'Unzip: {zip}', verbose)
console.log(f'To: {txt}', verbose) # TODO force to txt file
try:
with threading.Lock():
dir_txt = os.path.dirname(txt)
with ZipFile(zip, 'r') as z:
z.extractall(dir_txt)
except Exception as e:
console.log(vt.error('Unzip', e), verbose)
else:
console.log(vt.succes('Unzip'), verbose)
ok = True
return ok
def delete(fname, verbose=False):
'''Function deletes a file if exists'''
ok = False
console.log(f'Delete: {fname}', verbose)
if check(fname):
try:
Path(fname).unlink() # Remove file
except Exception as e:
console.log(vt.error('Delete', e), verbose)
else:
console.log(vt.succes('Delete'), verbose)
ok = True
else:
console.log(f'Delete cannot. File does not exist', verbose)
return ok
def mk_dir(path, verbose=False):
'''Function makes a map if not already exists'''
ok = False
console.log(f'Make dir: {path}', verbose)
try:
if os.path.isdir(path):
console.log('Map not made because it already exists.', verbose)
return True
else:
os.makedirs(path)
except Exception as e:
console.log(vt.error('Make directory', e), verbose)
else:
console.log(vt.succes('Make directory'), verbose)
ok = True
return ok
