def merge_pipeline(b, n): """ Merge Two pipeline files base on the above merge function """ # Merge my_merger = Merger( # pass in a list of tuples,with the # strategies you are looking to apply # to each type. [ (list, [merge_list]), (dict, ["merge"]) ], # next, choose the fallback strategies, # applied to all other types: ["override"], # finally, choose the strategies in # the case where the types conflict: ["override"] ) my_merger.merge(b,n) return b
def update_config(conf: dict, extra: dict) -> dict: def _update_item(c, k, v): if "." in k: tokens = k.split(".") current_k, remain_k = tokens[0], ".".join(tokens[1:]) c.setdefault(current_k, dict()) _update_item(c[current_k], remain_k, v) else: c[k] = v return if "__BASE__" in conf.keys(): base_path = conf["__BASE__"] conf.pop("__BASE__") if not os.path.exists(base_path): # __file__: quant-prob/quant_pack/utils/misc.py project_root = Path(__file__).absolute().parents[2] base_path = os.path.join(project_root, base_path) base_conf = yaml.load(open(base_path, "r", encoding="utf-8"), Loader=yaml.SafeLoader) else: base_conf = {} # strategies for merging configs: # - override lists in BASE by the counterparts in current config, such that # each param-group can get the latest, non-duplicated config; # - merge dicts recursively, such that derived config can be written as concisely as possible; conf_merger = Merger([(list, "override"), (dict, "merge")], ["override"], ["override"]) conf_merger.merge(base_conf, conf) for k, v in extra.items(): _update_item(base_conf, k, v) return base_conf
def _merge_config(left_config, right_config): merger = Merger( [ (list, ["append"]), (dict, ["merge"]) ], ["override"], ["override"] ) merged_config = deepcopy(left_config) merger.merge(merged_config, right_config) return merged_config
def merge(base: config_dict, overrides: config_dict, allow_in_place: bool = True) -> config_dict: merger = Merger( [(dict, ["merge"])], # strategies to use for each type ["override"], # fallback strategies applied to all other types ["override"] # strategies when the types conflict ) if not allow_in_place: base = deepcopy(base) overrides = deepcopy(overrides) merger.merge(base, overrides) return base
def merger(ref_path, overriding_jcasc_yaml): # load the jcasc defaults bundles int he docker image with open(f'{ref_path}/defaults.yaml') as file: default_jcasc = yaml.load(file, Loader=yaml.FullLoader) # load the user supplied jcasc with open(overriding_jcasc_yaml) as file: overriding_jcasc = yaml.load(file, Loader=yaml.FullLoader) # set up the merger with the merge stratigies jcasc_merger = Merger( [(list, ["override"]), (dict, [jcasc_dict_merge]), (set, ["union"])], # fallback strategy ["override"], # conflict strategy ["override"]) merged = jcasc_merger.merge(default_jcasc, overriding_jcasc) merged_jcasc = f'{ref_path}/jenkins.yaml' # write our merged jcasc yaml write_yaml_file(merged_jcasc, merged) return merged_jcasc
def parse_consul(value): consul_config = json.loads(os.environ['CONSUL_CONFIG']) if 'CONSUL_CONFIG' in os.environ else {} scheme = consul_config['scheme'] if 'scheme' in consul_config else 'http' host = consul_config['host'] if 'host' in consul_config else '127.0.0.1' port = consul_config['port'] if 'port' in consul_config else 8500 token = consul_config['token'] if 'token' in consul_config else None consul_merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'], ['override']) try: consul = Consul(scheme=scheme, host=host, port=port, token=token) _, kv_entries = consul.kv.get(recurse=True, key=value) except ACLPermissionDenied: print(BRIGHT_RED + 'Access denied connecting to: {}://{}:{}'.format(scheme, host, port) + RESET_ALL) return ERROR if not kv_entries: # Mark as failed if we can't find the consul key return ERROR consul_dict = {} for entry in kv_entries: subkeys = entry['Key'].split('/') value = entry['Value'].decode('utf-8') if hasattr(entry['Value'], 'decode') else entry['Value'] value = '' if value is None else value if '/' in entry['Key']: key = '{"' + entry['Key'].replace('/', '":{"') + '": "' + value + '"}'.ljust(len(subkeys)+1, '}') consul_dict = consul_merger.merge(consul_dict, json.loads(key)) else: consul_dict[entry['Key']] = value # return subkeys relative to rootkey rootkey = list(consul_dict.keys())[0] return consul_dict[rootkey]
def merge_dicts(base: dict, other: dict, list_semantics='set_merge'): ''' merges copies of the given dict instances and returns the merge result. The arguments remain unmodified. However, it must be possible to copy them using `copy.deepcopy`. Merging is done using the `deepmerge` module. In case of merge conflicts, values from `other` overwrite values from `base`. By default, different from the original implementation, a "set-merge" will be applied to lists. This results in deduplication and potential change of element order, which may be undesired. In this case, set `list_semantics` to 'None' ''' ensure_not_none(base) ensure_not_none(other) from deepmerge import Merger if list_semantics == 'set_merge': # monkey-patch merge-strategy for lists list_merge_strategy = Merger.PROVIDED_TYPE_STRATEGIES[list] list_merge_strategy.strategy_merge = lambda c, p, base, other: list( set(base) | set(other)) strategy_cfg = [(list, ['merge']), (dict, ['merge'])] merger = Merger(strategy_cfg, ['override'], ['override']) from copy import deepcopy # copy dicts, so they remain unmodified return merger.merge(deepcopy(base), deepcopy(other))
def load_config(config_file: Union[str, Path]) -> Dict: """ Loads and validates YAML config for this exporter and fills in default values :param config_file: :return: config as dictionary """ yaml = YAML() with open(config_file) as f: config = yaml.load(f) # merge with default config: use 'override' for lists to let users replace extractor setting entirely merger = Merger([(list, "override"), (dict, "merge")], ["override"], ["override"]) config = merger.merge(DEFAULT_CONFIG, config) for param in [IP_ADDRESS, PASSWORD]: if not param in config: raise ValueError( f"'{param}' is a mandatory config parameter, but it is missing in the YAML configuration file. Please see README.md for an example." ) if config[EXPORTER][TIMEOUT_SECONDS] <= 0: raise ValueError(f"'{TIMEOUT_SECONDS} must be positive.") if config[EXPORTER][PORT] < 0 or config[EXPORTER][PORT] > 65535: raise ValueError(f"Invalid exporter port.") if not config[EXPORTER][EXTRACTORS]: raise ValueError( "The config file needs to specify at least one family of metrics.") config[EXPORTER][EXTRACTORS] = sorted(set(config[EXPORTER][EXTRACTORS])) return config
def merge_value(reference, new_value): merger = Merger([(list, ["append"]), (dict, ["merge"])], ["override"], ["override"]) if isinstance(new_value, (list, set, dict)): new_reference = merger.merge(reference, new_value) else: raise TypeError("Cannot handle merge_value of type {}".format(type(new_value))) return new_reference
def get_section(section: str) -> Dict[str, Any]: """ Load the section config from config-default.toml and config.toml. Use deepmerge.Merger to merge the configuration from config.toml and override that of config-default.toml. """ # Load default configuration if not Path("config-default.toml").exists(): raise MetricityConfigurationError("config-default.toml is missing") with open("config-default.toml", "r") as default_config_file: default_config = toml.load(default_config_file) # Load user configuration user_config = {} if Path("config.toml").exists(): with open("config.toml", "r") as default_config_file: user_config = toml.load(default_config_file) # Merge the configuration merger = Merger([(dict, "merge")], ["override"], ["override"]) conf = merger.merge(default_config, user_config) # Check whether we are missing the requested section if not conf.get(section): raise MetricityConfigurationError( f"Config is missing section '{section}'") return conf[section]
def main(): root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) doc = {} doc['openapi'] = '3.0.1' doc['info'] = { 'title': 'NBA Stats API', 'description': 'OpenAPI documentation for the NBA Stats API generated by sigmer/nba-stats-openapi', 'version': __version__ } doc['servers'] = [{'url': base_url + base_path}] for endpoint in endpoints: handle_endpoint(doc, endpoint) # Uncomment to include header parameters # for key, value in headers.items(): # components = doc['components'] # parameters = components['parameters'] # header = { # 'name': key, # 'in': 'header', # 'required': True, # 'schema': {'type': 'string'}, # 'example': value # } # parameters[key] = header # components['parameters'] = parameters # doc['components'] = components default = {} with open('/'.join([root_dir, 'templates', 'default.yaml'])) as f: default = yaml.load(f, Loader=yaml.Loader) merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'], ['override']) merger.merge(doc, default) dist_dir = '/'.join([root_dir, 'dist/']) os.makedirs(os.path.dirname(dist_dir), exist_ok=True) with open(dist_dir + 'swagger.yaml', 'w') as output: yaml.dump(doc, output, sort_keys=False)
def _update_cfg_with_base(cfg): if cfg["__BASE__"]: base_path = cfg.pop("__BASE__") if not os.path.exists(base_path): # __file__: quant_pack/apis/env.py project_root = Path(__file__).absolute().parents[2] base_path = os.path.join(project_root, base_path) base_cfg = yaml.load(open(base_path, "r", encoding="utf-8"), Loader=yaml.SafeLoader) base_cfg = _update_cfg_with_base(base_cfg) # strategies for merging configs: # - override lists in BASE by the counterparts in current config, such that # each param-group can get the latest, non-duplicated config; # - merge dicts recursively, such that derived config can be written as concisely as possible; conf_merger = Merger([(list, "override"), (dict, "merge")], ["override"], ["override"]) conf_merger.merge(base_cfg, cfg) cfg = base_cfg return cfg
class GraphGenerator(object): """Convert a list of key-value luples into a graph-like hash. e.g [ ('key1', 'value1'), ('key2', value2') ] State is persisted to disk in JSON format """ def __init__(self, filename='graph.json'): self.filename = filename self.dict_merger = Merger([(list, ["append"]), (dict, ["merge"])], ["override"], ["override"]) self.load() def __del__(self): self.persist() def load(self): '''Load graph from file''' try: with open(self.filename) as f: self.graph = json.load(f) except IOError: self.graph = {} def persist(self): ''' Persist graph to file''' with open('{}.new'.format(self.filename), 'w') as json_output: json.dump(self.graph, json_output) os.rename('{}.new'.format(self.filename), self.filename) def add_list(self, list): """ list = [ ('key1', 'value1'), ('key2', 'value2')...('keyN', 'valueN') ] """ new_graph = self._parse_list(list) self.dict_merger.merge(self.graph, new_graph) def _parse_list(self, list): try: key, value = list.pop(0) return self._generate_node(key, value, self._parse_list(list)) except IndexError: return {} def _generate_node(self, key, value, edges): return {key: {'value': value, '_edges': edges}}
def merge(*dicts): result_dict = {} merger = Merger( # merge strategies - for each type. [ (dict, ['merge']), (list, ['override']), ], # fallback strategies - for all other types ['override'], # conflict strategies ['override'], ) for next_dict in dicts: merger.merge(result_dict, next_dict) return result_dict
def merge_dicts(base: dict, *other: dict, list_semantics='merge'): ''' merges copies of the given dict instances and returns the merge result. The arguments remain unmodified. However, it must be possible to copy them using `copy.deepcopy`. Merging is done using the `deepmerge` module. In case of merge conflicts, values from `other` overwrite values from `base`. By default, different from the original implementation, a merge will be applied to lists. This results in deduplication retaining element order. The elements from `other` are appended to those from `base`. ''' not_none(base) not_empty(other) from deepmerge import Merger if list_semantics == 'merge': # monkey-patch merge-strategy for lists list_merge_strategy = Merger.PROVIDED_TYPE_STRATEGIES[list] list_merge_strategy.strategy_merge = lambda c, p, base, other: \ list(base) + [e for e in other if e not in base] strategy_cfg = [(list, ['merge']), (dict, ['merge'])] merger = Merger(strategy_cfg, ['override'], ['override']) elif list_semantics is None: strategy_cfg = [(dict, ['merge'])] merger = Merger(strategy_cfg, ['override'], ['override']) else: raise NotImplementedError from copy import deepcopy return functools.reduce( lambda b, o: merger.merge(b, deepcopy(o)), [base, *other], {}, )
courses.remove(cls.replace('!', '')) courses += [i for i in bundle if not i.startswith('!')] return classes config = {} config_files = [ 'config_1', 'config_2', 'config_3', 'config_6' ] for c in config_files: merger.merge(config, yaml.load(open('configs/%s.yml' % (c,), 'r'))) config['tclasses'] = transform_classes(config) config['blocks'] = config['blocks'] if 'blocks' in config else {} scen = Scenario('Schedule', horizon=(DAYS * 5)) # Will be created when needed teachers = {} classes = {} for cls, _ in config['tclasses'].items(): classes[cls] = scen.Resource(cls) courses = {} for course, info in config['courses'].items():
print('Parsing P-factor data...') pfactor_data_dict = du.parse_pfactor_data(config['p-factor'], data_dict) datasets.append(pfactor_data_dict) print('Parsing precipitation data...') precip_data_dict = du.parse_precipitation_data(config['precipitation_dir'], data_dict, int(config['timesteps'])) datasets.append(precip_data_dict) # Merge the datasets print('Merging datasets...') my_merger = Merger([(list, ["override"]), (dict, ["merge"])], ["override"], ["override"]) for dataset in datasets: data_dict = my_merger.merge(data_dict, dataset) # Converting RiverReaches to EstuaryReaches, where applicable print('Setting tidal bounds...') data_dict = du.parse_tidal_bounds(config['tidal_bounds'], data_dict) # Filling in the gaps print('Filling the gaps...') data_dict["dimensions"].update({"d": 2, "state": 7, "form": 4, "n": 5}) for grid_cell_ref, grid_cell in data_dict.items(): if grid_cell_ref not in [ "dimensions", "grid_dimensions[d]", "routed_reaches[branches][seeds][river_reach_ref]" ]: x = grid_cell['x'] y = grid_cell['y']
class BaseDBHandler(object): """Base handler for db.""" __version__ = 'v4' db_defaults = load_config(__version__).db.connection.base _df_class = 'base_df' _config = AttrDict() _df_name = 'base_df' _columns = None def __new__(cls, db_class: str = None, db_engine: str = None, **kwargs) -> object: """Create object. Args: db_class (str): database class (e.g. 'meta') db_engine (str): database engine (e.g. 'sqlite') **kwargs: DB-handler specific arguments Returns: (object): the corresponding handler object """ if cls is BaseDBHandler: handler = cls._get_handler(db_class, db_engine) return super(BaseDBHandler, cls).__new__(handler) else: return super(BaseDBHandler, cls).__new__(cls) @classmethod def _get_handler(cls, db_class, db_engine=None): """Returns an appropriate handler. Args: db_class (str): database class (e.g. 'meta') db_engine (str): database engine (e.g. 'tinydb') Returns: (handler): database handler object """ # Load default config config = load_config(cls.__version__) # Check if the db_class is available if db_class not in DB_HANDLERS.keys(): raise ValueError('Unsupported db_class: {}'.format(db_class)) # Get db_engine from environment variable if not specified if db_engine is None: db_engine = os.environ.get( 'PYDTK_{}_DB_ENGINE'.format(db_class.upper()), None) # Get the default engine if not specified if db_engine is None: try: db_defaults = getattr(config.db.connection, db_class) db_engine = db_defaults.engine except (ValueError, AttributeError): raise ValueError( 'Could not find the default value for `db_engine`') # Check if the corresponding handler is registered if db_engine not in DB_HANDLERS[db_class].keys(): raise ValueError('Unsupported db_engine: {}'.format(db_engine)) # Get a DB-handler supporting the engine return DB_HANDLERS[db_class][db_engine] def __init__(self, db_engine=None, db_host=None, db_name=None, db_username=None, db_password=None, df_name=None, read_on_init=False, **kwargs): """Initialize BaseDBHandler. Args: db_engine (str): database engine (if None, the one in the config file will be used) db_host (str): database HOST (if None, the one in the config file will be used) db_name (str): database name (if None, the one in the config file will be used) db_username (str): username (if None, the one in the config file will be used) db_password (str): password (if None, the one in the config file will be used) df_name (str): dataframe (table in DB) name (if None, class default value will be used) read_on_init (bool): if True, dataframe will be read from database on initialization """ super(BaseDBHandler, self).__init__() self.logger = logging.getLogger(__name__) self._cursor = 0 self._data = {} self._uuids_to_remove = [] self._count_total = 0 if df_name is not None: self.df_name = df_name # Load config config = load_config(self.__version__) try: self._config = ConfigDict(config['db']['df_class'][self._df_class]) except KeyError: self._config = ConfigDict() # Initialize deepmerger self._merger = Merger( # pass in a list of tuple, with the # strategies you are looking to apply # to each type. [(dict, ['merge']), (list, [_deepmerge_append_list_unique, 'append'])], # next, choose the fallback strategies, # applied to all other types: ["override"], # finally, choose the strategies in # the case where the types conflict: ["override"]) # Initialize database self._initialize_engine(db_engine, db_host, db_name, db_username, db_password) self._load_config_from_db() # Fetch table if read_on_init: self.read() def __len__(self): """Return number of recoreds.""" return len(self.data) def __iter__(self): """Return iterator.""" return self def __next__(self): """Return the next item.""" if self._cursor >= len(self): self._cursor = 0 raise StopIteration() # Grab data data = self[self._cursor] # Increment self._cursor += 1 return data def __getitem__(self, idx, remove_internal_columns=True): """Return data at index idx. Args: idx (int): index of the target data remove_internal_columns (bool): if True, internal columns are removed Returns: (dict): data """ data = self.data[idx] # Delete internal column if remove_internal_columns: if '_uuid' in data.keys(): del data['_uuid'] if '_creation_time' in data.keys(): del data['_creation_time'] return data def _initialize_engine(self, db_engine=None, db_host=None, db_name=None, db_username=None, db_password=None): """Initialize DB engine. Args: db_engine (str): database engine (if None, the one in the config file will be used) db_host (str): database HOST (if None, the one in the config file will be used) db_name (str): database name (if None, the one in the config file will be used) db_username (str): username (if None, the one in the config file will be used) db_password (str): password (if None, the one in the config file will be used) """ if db_engine is None: db_engine = self.db_defaults.engine if db_host is None: db_host = self.db_defaults.host if db_username is None: db_username = self.db_defaults.username if db_password is None: db_password = self.db_defaults.password self._db_engine = db_engine if db_engine in DB_ENGINES.keys(): self._db = DB_ENGINES[db_engine].connect( db_host=db_host, db_name=db_name, db_username=db_username, db_password=db_password, collection_name=self._df_name, handler=self, ) self._config_db = DB_ENGINES[db_engine].connect( db_host=db_host, db_name=db_name, db_username=db_username, db_password=db_password, collection_name='--config--{}'.format(self._df_name), handler=self, ) else: raise ValueError("Unsupported engine: {}".format(db_engine)) def _load_config_from_db(self): """Load configs from DB.""" if self._db_engine not in DB_ENGINES.keys(): return try: candidates = DB_ENGINES[self._db_engine].read(self._config_db, handler=self) if len(candidates) > 0: if isinstance(candidates[0][0], dict): self._config = ConfigDict(candidates[0][0]) else: raise TypeError('Unexpected type') except Exception as e: logging.warning('Failed to load configs from DB: {}'.format( str(e))) def _save_config_to_db(self): """Save configs to DB.""" if self._db_engine not in DB_ENGINES.keys(): return try: config = dict(self._config) config.update({'_uuid': '__config__'}) config = [config] DB_ENGINES[self._db_engine].upsert(self._config_db, data=config, handler=self) except Exception as e: logging.warning('Failed to save configs to DB: {}'.format(str(e))) def _get_uuid_from_item(self, data_in): """Return UUID of the given item. Args: data_in (dict or pandas.Series): dict or Series containing data Returns: (str): UUID """ hash_target_columns = \ self._config['index_columns'] if 'index_columns' in self._config.keys() else [] item = data_in if isinstance(item, pd.Series): item = data_in.to_dict() pre_hash = ''.join([ '{:.09f}'.format(item[column]) if isinstance(item[column], float) else str(item[column].keys()) if isinstance(item[column], dict) else str(item[column]) for column in hash_target_columns if column in item.keys() ]) pre_hash = pre_hash.encode('utf-8') uuid = hashlib.md5(pre_hash).hexdigest() return uuid def _read(self, **kwargs): if self._db_engine is None: raise DatabaseNotInitializedError() elif self._db_engine in DB_ENGINES.keys(): func = DB_ENGINES[self._db_engine].read available_args = set(inspect.signature(func).parameters.keys()) unavailable_args = set([k for k, v in kwargs.items() if v is not None]) \ .difference(available_args) if len(unavailable_args) > 0: logging.warning( 'DB-engine "{0}" does not support args: {1}'.format( self._db_engine, list(unavailable_args))) return func(self._db, handler=self, **kwargs) else: raise ValueError('Unsupported DB engine: {}'.format( self._db_engine)) def read(self, df_name=None, query=None, pql=None, where=None, group_by=None, order_by=None, limit=None, offset=None, **kwargs): """Read data from SQL. Args: df_name (str): Dataframe name to read query (str SQL query or SQLAlchemy Selectable): query to select items pql (PQL): Python-Query-Language to select items where (str): query string for filtering items group_by (str): column name to group order_by (srt): column name to sort by limit (int): number of items to return per a page offset (int): offset of cursor **kwargs: kwargs for function `pandas.read_sql_query` or `influxdb.DataFrameClient.query` """ if df_name is not None: self.df_name = df_name data, self._count_total = self._read(query=query, pql=pql, where=where, group_by=group_by, order_by=order_by, limit=limit, offset=offset, **kwargs) # Check if UUID exists in each value for value in data: if '_uuid' not in value.keys(): raise ValueError('"_uuid" not found in data') self._data = {record['_uuid']: record for record in data} def _upsert(self, data): """Upsert data to DB. Args: data (list): data to save """ if self._db_engine is None: raise DatabaseNotInitializedError() elif self._db_engine in DB_ENGINES.keys(): DB_ENGINES[self._db_engine].upsert(self._db, data) else: raise ValueError('Unsupported DB engine: {}'.format( self._db_engine)) def save(self, **kwargs): """Save data to DB.""" self._remove(self._uuids_to_remove) self._uuids_to_remove = [] self._upsert(list(self._data.values())) self._save_config_to_db() def _remove(self, uuids): """Remove data from DB. Args: uuids [str]: A list of unique IDs """ if self._db_engine is None: raise DatabaseNotInitializedError() elif self._db_engine in DB_ENGINES.keys(): DB_ENGINES[self._db_engine].remove(self._db, uuids) else: raise ValueError('Unsupported DB engine: {}'.format( self._db_engine)) def _drop_table(self, name): """Drop table from DB. Args: name (str): Name of table (collection) """ if self._db_engine is None: raise DatabaseNotInitializedError() elif self._db_engine in DB_ENGINES.keys(): DB_ENGINES[self._db_engine].drop_table(self._db, name) else: raise ValueError('Unsupported DB engine: {}'.format( self._db_engine)) def add_data(self, data_in, strategy='overwrite', **kwargs): """Add data to db. Args: data_in (dict): a dict containing data strategy (str): 'merge' or 'overwrite' """ assert strategy in ['merge', 'overwrite'], 'Unknown strategy.' data = deepcopy(data_in) if '_uuid' not in data.keys() or '_creation_time' not in data.keys(): # Add _uuid and _creation_time data['_uuid'] = self._get_uuid_from_item(data) data['_creation_time'] = datetime.now().timestamp() if data['_uuid'] in self._data.keys(): if strategy == 'merge': base_data = self._data[data['_uuid']] data = self._merger.merge(base_data, data) # Add new columns (keys) to config columns = self._config['columns'] if 'columns' in self._config.keys( ) else [] columns_existing = [c['name'] for c in columns] columns_in_data = list(data_in.keys()) new_columns = [] for new_column in set(columns_in_data).difference(columns_existing): name = new_column dtype = type(data_in[new_column]).__name__ aggregation = 'first' new_columns.append({ 'name': name, 'dtype': dtype, 'aggregation': aggregation, 'display_name': name, }) columns += new_columns self._config['columns'] = columns self._data.update({data['_uuid']: data}) def remove_data(self, data): """Remove data-record from DB. Args: data (dict): a dict containing the target data or '_uuid' in keys. """ if '_uuid' in data.keys(): uuid = data['_uuid'] else: uuid = self._get_uuid_from_item(data) # Remove from in-memory data if uuid in self._data.keys(): del self._data[uuid] # Remove from DB on saving self._uuids_to_remove.append(uuid) def _df_from_dicts(self, dicts): """Create a DataFrame from a list of dicts. Args: dicts (list): list of dicts Returns: (pd.DataFrame): a data-frame """ columns = self._config['columns'] if 'columns' in self._config.keys( ) else [] df = pd.concat( [ pd.Series(name=c['name'], dtype=dtype_string_to_dtype_object(c['dtype'])) for c in columns if c['name'] != '_uuid' and c['name'] != '_creation_time' ] # noqa: E501 + [ pd.Series(name='_uuid', dtype=str), pd.Series(name='_creation_time', dtype=float) ], axis=1) df.set_index('_uuid', inplace=True) df = pd.concat([df, pd.DataFrame.from_records(dicts)]) return df def _to_display_names(self, df, inplace=False): """Rename columns to display-names. Args: df (pd.DataFrame): A data-frame inplace (bool): If true, df will be modified in-place. Returns: (pd.DataFrame): A data-frame with renamed columns if inplace=false, otherwise None. """ column_renames = { c['name']: c['display_name'] for c in self._config['columns'] if isinstance(c, dict) and 'name' in c.keys() and 'display_name' in c.keys() } return df.rename(columns=column_renames, inplace=inplace) @property def data(self): """Return data. Returns: (list): list of dicts """ return list(self._data.values()) @data.setter def data(self, data): """Setter for self.data. Args: data (list): new-data """ assert isinstance(data, list) for value in data: self.add_data(value) @property def columns(self): """Return columns of DF.""" df = self._df_from_dicts(self.data) return df.columns.tolist() @property def df(self): """Return df.""" df = self._df_from_dicts(self.data) df = self._to_display_names(df) return df @property def count_total(self): """Return total number of rows.""" return self._count_total @property def config(self): """Return config.""" return self._config
class EZGrow(): def __init__(self): self.conf = dict() self.merger = Merger([(list, ["append"]), (dict, ["merge"])], ["override"], ["override"]) for c in ['/etc/ezgrow/ezgrow.json', '/site/etc/ezgrow/ezgrow.json']: try: with open(c, 'r') as s: self.merger.merge(self.conf, json.load(s)) except Exception as e: print("Can't load config %s: [%s]" % (c, e)) if len(self.conf) == 0: raise RuntimeError('Config is empty, NOT rebooting the host') def reload_sensors(self): value = dict() for f in self.conf['json']: try: with open(f) as s: self.merger.merge(value, json.load(s)) except Exception as e: print("Can't load sensor %s: [%s]" % (f, e)) return value def update_watchdog(self): try: self.watchdog.write('X') self.watchdog.flush() except AttributeError: print("Opening watchdog device %s" % self.conf['watchdog']) self.watchdog = open(self.conf['watchdog'], 'w') def reload_gpio(self): status, gpio = getstatusoutput('gpio allreadall') assert (not status), 'Failed to run gpio' value = dict() for line in filter(lambda x: len(x) > 10 and x[1][0].isdigit(), map(lambda x: x.split(None), gpio.splitlines())): value[int(line[1])] = 1 if line[5] == 'High' else 0 # col 1 value[int(line[8])] = 1 if line[12] == 'High' else 0 # col 2 return value def reload_snmp(self): on_value = self.conf['snmp']['value']['on'] return dict(map(lambda x: (x[0], on_value == self.snmp_get(x[1])), \ self.conf['snmp']['oid'].items())) def snmp_get(self, oid): status,snmp = getstatusoutput('snmpget -v1 -Oq -c%s %s %s' % \ (self.conf['snmp']['pass'], self.conf['snmp']['addr'], oid) ) assert (not status), 'Failed to run snmpget' for line in map(lambda x: x.split(None), snmp.splitlines()): return line[1] def snmp_set(self, oid, kind, value): status,snmp = getstatusoutput('snmpset -v1 -Oq -c%s %s %s %s %s' % \ (self.conf['snmp']['pass'], self.conf['snmp']['addr'], oid, kind, value) ) assert (not status), 'Failed to run snmpset' return self.snmp_get(oid) def avg(self): # return [min, max, avg] value, count = [float('inf'), float('-inf'), 0.0], 0.0 for inside in self.conf['inside-temperature']: for sensor in filter(lambda x: x.startswith(inside), self.conf['tempdata']): value[0] = min( value[0], float(self.conf['tempdata'][sensor]['temperature'])) value[1] = max( value[1], float(self.conf['tempdata'][sensor]['temperature'])) value[2] += float(self.conf['tempdata'][sensor]['temperature']) count += 1.0 value[2] = value[2] / count if count != 0 else float('inf') return value def _any_low(self, name): # returns True if any pin is low for pin in self.conf['gpio'][name]: #print('Checking pin (l): %d' % pin) if self.conf['gpiodata'][pin] == 0: return True return False def _any_high(self, name): # returns True if any pin is high for pin in self.conf['gpio'][name]: #print('Checking pin (h): %d' % pin) if self.conf['gpiodata'][pin] != 0: return True return False def get_pump(self): return not(self._any_low('leak') \ or self._any_high('water-high')) def get_lamp(self): tm = datetime.strptime(self.conf['timestamp'].strftime('%H:%M'), '%H:%M') t = self.conf['time'][self.conf['time'] ['mode']] # { 'on': ..., 'off': ... } return not ((tm > datetime.strptime(t['off'], '%H:%M')) \ and (tm < datetime.strptime(t['on'], '%H:%M'))) def get_ozone(self): # TODO use holiday module and make sure don't turn on ozone if self.conf['timestamp'].strftime('%a') in ['Sat', 'Sun']: return False h = int(self.conf['timestamp'].strftime('%H')) m = int(self.conf['timestamp'].strftime('%M')) # 1000 to 1100 and 10 minutes every hour until 1800 if h == 10 or (h > 8 and h < 19 and m < 15): return True return False def get_fan_ext(self): avg = self.avg()[1] # get maximum value if avg in [float('inf'), float('-inf')]: return True # safety net; turn on if no sensors fan = self.conf['snmpdata']['fan-ext'] if fan: # fan is on now if avg < self.conf['temp']['off']: return False else: # fan is off now if avg > self.conf['temp']['on']: return True return fan != 0 # keep fan unchanged def run(self): for group in self.conf['gpio']: for pin in self.conf['gpio'][group]: status, gpio = getstatusoutput('gpio -g mode %d in' % pin) assert (not status), 'Failed to run gpio mode in' status, gpio = getstatusoutput('gpio -g mode %d down' % pin) assert (not status), 'Failed to run gpio mode down' while True: # used for system schedulig (lamps, etc.) self.conf['timestamp'] = datetime.now() self.update_watchdog() self.conf['gpiodata'] = self.reload_gpio() self.update_watchdog() self.conf['tempdata'] = self.reload_sensors() self.update_watchdog() self.conf['snmpdata'] = self.reload_snmp() self.update_watchdog() self.conf['update'] = { 'lamp': self.get_lamp(), # this only depend on timer 'pump': self.get_pump(), # this only depend on sensors 'fan-ext': self.get_fan_ext(), 'fan-int': True, # TODO some logic 'ozone': self.get_ozone(), } print(self.conf['update']) for outlet in self.conf['update'].items(): self.update_watchdog() if outlet[1] == self.conf['snmpdata'][outlet[0]]: continue oid = self.conf['snmp']['oid'][outlet[0]] value = self.conf['snmp']['value'][ 'on' if outlet[1] else 'off'] self.snmp_set(oid, 'i', value) tm = datetime.now() # update timestamp self.conf['timestamp'] = tm # '%s' % tm # is not JSON serializable with open(tempfile.gettempdir() + '/status.yaml', 'w') as s: pyaml.dump(self.conf, s) # , indent=4, sort_keys=True) time.sleep(self.conf['sleep'] - tm.second % self.conf['sleep'])