def fetch_DeBilt_weather(): """ Downloads, cleans and pickles weather data from the KNMI website. """ home = 'http://www.knmi.nl/' url = home + 'klimatologie/daggegevens/datafiles3/260/etmgeg_260.zip' path = get_data_dir() file = process_zip(url, path, 'etmgeg_260.txt') df = pd.read_csv( file, skiprows=47, usecols=['YYYYMMDD', 'DDVEC', ' FG', ' TG', ' RH', ' PG'], index_col=0, parse_dates=True, na_values=' ') df.columns = ['WIND_DIR', 'WIND_SPEED', 'TEMP', 'RAIN', 'PRESSURE'] df[df['RAIN'] == -1] = 0.05 / 2 df['WIND_SPEED'] = 0.1 * df['WIND_SPEED'] df['TEMP'] = 0.1 * df['TEMP'] df['RAIN'] = 0.1 * df['RAIN'] df[df['PRESSURE'] < 1] = np.nan df['PRESSURE'] = 0.1 * df['PRESSURE'] log_api.conf_logger(__name__).warning(df.index[-1]) pckl = os.path.join(path, 'weather.pckl') df.to_pickle(pckl) assert df.equals(pd.read_pickle(pckl))
def write(self, fname): """ Writes to a file. :param fname: The name of the file. """ with open(fname, 'w') as f: f.write(self.rst) log_api.conf_logger(__name__).warning('Wrote to ' + fname)
def download(url, out): """ Download a file from the web. :param url: The URL of the file. :param out: The path of the file. """ req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) res = urlrequest.urlopen(req) with open(out, 'wb') as data_file: data_file.write(res.read()) log_api.conf_logger(__name__).warning('Downloaded ' + data_file.name)
def get_data_dir(): """ Finds the appropriate data directory to store data files. :returns: A data directory, which is OS dependent. """ dirs = AppDirs(options.APP_DIR, "Ivan Idris") path = dirs.user_data_dir log_api.conf_logger(__name__).warning('Data dir ' + path) if not os.path.exists(path): os.mkdir(path) return path
def __init__(self, dir): self.dir = os.path.join(data.get_data_dir(), dir) self.LOG = log_api.conf_logger(__name__) if not os.path.exists(self.dir): os.mkdir(self.dir) self.csv_fname = os.path.join(self.dir, 'metadata.csv') self.url_set = set() self.init_url_csv()
def __init__(self, nrows, ncols, val): self.nrows = nrows self.ncols = ncols self.grid = [ncols * [val] for i in range(nrows)] self.logger = log_api.conf_logger('collect.GridList')
def generate(self): self.add_css() self.gen.write() log_api.conf_logger(__name__).warning('Generated HTML in ' + self.out_file)