def fetch_DeBilt_weather():
""" Downloads, cleans and pickles weather data from the
KNMI website. """
home = 'http://www.knmi.nl/'
url = home + 'klimatologie/daggegevens/datafiles3/260/etmgeg_260.zip'
path = get_data_dir()
file = process_zip(url, path, 'etmgeg_260.txt')
df = pd.read_csv(
file,
skiprows=47,
usecols=['YYYYMMDD', 'DDVEC', ' FG', ' TG', ' RH', ' PG'],
index_col=0,
parse_dates=True,
na_values=' ')
df.columns = ['WIND_DIR', 'WIND_SPEED', 'TEMP', 'RAIN', 'PRESSURE']
df[df['RAIN'] == -1] = 0.05 / 2
df['WIND_SPEED'] = 0.1 * df['WIND_SPEED']
df['TEMP'] = 0.1 * df['TEMP']
df['RAIN'] = 0.1 * df['RAIN']
df[df['PRESSURE'] < 1] = np.nan
df['PRESSURE'] = 0.1 * df['PRESSURE']
log_api.conf_logger(__name__).warning(df.index[-1])
pckl = os.path.join(path, 'weather.pckl')
df.to_pickle(pckl)
assert df.equals(pd.read_pickle(pckl))
def write(self, fname):
""" Writes to a file.
:param fname: The name of the file.
"""
with open(fname, 'w') as f:
f.write(self.rst)
log_api.conf_logger(__name__).warning('Wrote to ' + fname)
def download(url, out):
""" Download a file from the web.
:param url: The URL of the file.
:param out: The path of the file.
"""
req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
res = urlrequest.urlopen(req)
with open(out, 'wb') as data_file:
data_file.write(res.read())
log_api.conf_logger(__name__).warning('Downloaded ' + data_file.name)
def get_data_dir():
""" Finds the appropriate data directory to store data files.
:returns: A data directory, which is OS dependent.
"""
dirs = AppDirs(options.APP_DIR, "Ivan Idris")
path = dirs.user_data_dir
log_api.conf_logger(__name__).warning('Data dir ' + path)
if not os.path.exists(path):
os.mkdir(path)
return path
def __init__(self, dir):
self.dir = os.path.join(data.get_data_dir(), dir)
self.LOG = log_api.conf_logger(__name__)
if not os.path.exists(self.dir):
os.mkdir(self.dir)
self.csv_fname = os.path.join(self.dir, 'metadata.csv')
self.url_set = set()
self.init_url_csv()
def __init__(self, nrows, ncols, val):
self.nrows = nrows
self.ncols = ncols
self.grid = [ncols * [val] for i in range(nrows)]
self.logger = log_api.conf_logger('collect.GridList')
def generate(self):
self.add_css()
self.gen.write()
log_api.conf_logger(__name__).warning('Generated HTML in ' +
self.out_file)