import pandas as pd import time import traceback try: from config_handler import handle_config from logger import Logger import utils except: print( "Failed to import config_handler, logger or utils.\n" + "They are located in ../tools/\n" ) print( traceback.format_exc() ) exit( 1 ) cp = handle_config( metadata={"invoking_script":"hist_inventory"}, header="inventory" ) logger = Logger( "hist_inventory", cp ) ### load configuration SITE = cp["site_id"] CAMERAS = cp["cameras"][SITE]["cameras"] NUM_CAMERAS = len(CAMERAS) # how long to sleep once we run out of images to log SLEEP_LEN = cp["inventory"]["hist"]["sleep_time"] ROOT = cp["paths"][SITE]["img_path"] CACHE_ROOT = cp["paths"][SITE]["cache_path"] FOLDER_NAMES = [ROOT + x + "/" for x in CAMERAS]
logger.info("Would retrieve {} to {}".format(url, fn)) else: try: dfs = pd.read_html(url, index_col=1, header=0) except: logger.error("{} could not be retrieved".format(url)) return df = dfs[0] try: df.to_csv(fn) except: logger.error("{} could not be written".format(fn)) if __name__ == "__main__": cp = handle_config(metadata={"invoking_script": "GHI_downloader"}, header="downloader") site = cp["site_id"] config = cp['downloader'] SAFE_MODE = config['safe_mode'] # run without consequences? if SAFE_MODE: print("Initializing GHI_downloader in safe_mode") GHI_suffix = config['GHI_suffix'] flush_interval = config["flush_interval"] interval_day = config['interval_day'] interval_night = config['interval_night'] # using site-specific config files on command line lat = config['geolocation']['lat'] lon = config['geolocation']['lon']
# create daily .csv files # Use standard config_handler with .yaml config to parse arguments from config_handler import handle_config from datetime import datetime, timezone import pandas as pd import pysolar.solar as ps import sys import time # note: pathlibs Path object have their own open method, use that for python <3.6 from pathlib import Path if __name__ == "__main__": cp = handle_config(metadata={"invoking_script": "GHIdaily"}, header="diagnostics") site = cp["site_id"] config = cp['downloader'] paths = cp['paths'] logpath = Path(paths['logging_path']) logpath.mkdir(exist_ok=True) GHIpath = Path(paths['raw_GHI_path']) start_date = int(cp['diagnostics']['start_date']) end_date = int(cp['diagnostics']['end_date']) force_overwrite = int(cp['diagnostics']['force_overwrite']) GHIsensors = {} for GHIsensor in sorted(cp['GHI_sensors'].keys()): GHIsensor = GHIsensor.upper() GHIsensors[GHIsensor] = cp['GHI_sensors'][GHIsensor] dest = Path(GHIpath, GHIsensor)