skip = 0 if counter is None else int(counter)
# Get MOSMIX stations
try:
stations = pd.read_csv(MOSMIX_PATH,
dtype='str',
skiprows=skip,
nrows=STATIONS_PER_CYCLE,
names=['id', 'mosmix'])
except pd.errors.EmptyDataError:
stations = None
pass
# Update counter
if stations is None or len(stations.index) < STATIONS_PER_CYCLE:
task.set_var('station_counter', 0)
exit()
else:
task.set_var('station_counter', skip + STATIONS_PER_CYCLE)
# DataFrame which holds all data
df_full = None
# Import data for each weather station
for station in stations.to_dict(orient='records'):
try:
# Load KMZ data from DWD server
url = f"https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/{station['mosmix']}/kml/MOSMIX_L_LATEST_{station['mosmix']}.kmz"
filename = os.path.dirname(__file__) + os.sep + station['id'] + '.kmz'
# Get counter value
counter = task.get_var(f'station_counter_{MODE}')
counter = int(counter) if counter is not None else 0
skip = 3 if counter is None else 3 + counter
# Get all files in directory
try:
endpos = STATIONS_PER_CYCLE + skip
stations = ftp.nlst()[skip:endpos]
except BaseException:
stations = None
pass
# Update counter
if stations is None or len(stations) < STATIONS_PER_CYCLE:
task.set_var(f'station_counter_{MODE}', 0)
exit()
else:
task.set_var(f'station_counter_{MODE}', counter + STATIONS_PER_CYCLE)
# DataFrame which holds all data
df_full = None
for station_file in stations:
try:
# Get national weather station ID
national_id = str(station_file[-13:-8]) if MODE == 'recent' else str(
station_file[-32:-27])
station = task.read(
# Get ISD Lite stations
try:
stations = pd.read_csv(USAF_WBAN_PATH,
dtype='str',
skiprows=skip,
nrows=STATIONS_PER_CYCLE,
names=['id', 'usaf', 'wban'])
except pd.errors.EmptyDataError:
stations = None
pass
# Update counter
if stations is None or len(stations.index) < STATIONS_PER_CYCLE:
# Reset counter
task.set_var('station_counter_' + MODE, 0)
# Reset year
if MODE == 'historical':
if year >= CURRENT_YEAR - 2:
task.set_var('year', 1901)
else:
task.set_var('year', year + 1)
exit()
else:
task.set_var('station_counter_' + MODE, skip + STATIONS_PER_CYCLE)
# Connect to NOAA FTP Server
ftp = FTP('ftp.ncdc.noaa.gov')
ftp.login()
# Get list of years