def merge_pipeline(b, n):
"""
Merge Two pipeline files base on the above merge function
"""
# Merge
my_merger = Merger(
# pass in a list of tuples,with the
# strategies you are looking to apply
# to each type.
[
(list, [merge_list]),
(dict, ["merge"])
],
# next, choose the fallback strategies,
# applied to all other types:
["override"],
# finally, choose the strategies in
# the case where the types conflict:
["override"]
)
my_merger.merge(b,n)
return b
def update_config(conf: dict, extra: dict) -> dict:
def _update_item(c, k, v):
if "." in k:
tokens = k.split(".")
current_k, remain_k = tokens[0], ".".join(tokens[1:])
c.setdefault(current_k, dict())
_update_item(c[current_k], remain_k, v)
else:
c[k] = v
return
if "__BASE__" in conf.keys():
base_path = conf["__BASE__"]
conf.pop("__BASE__")
if not os.path.exists(base_path):
# __file__: quant-prob/quant_pack/utils/misc.py
project_root = Path(__file__).absolute().parents[2]
base_path = os.path.join(project_root, base_path)
base_conf = yaml.load(open(base_path, "r", encoding="utf-8"),
Loader=yaml.SafeLoader)
else:
base_conf = {}
# strategies for merging configs:
# - override lists in BASE by the counterparts in current config, such that
# each param-group can get the latest, non-duplicated config;
# - merge dicts recursively, such that derived config can be written as concisely as possible;
conf_merger = Merger([(list, "override"), (dict, "merge")], ["override"],
["override"])
conf_merger.merge(base_conf, conf)
for k, v in extra.items():
_update_item(base_conf, k, v)
return base_conf
def _merge_config(left_config, right_config):
merger = Merger(
[
(list, ["append"]),
(dict, ["merge"])
],
["override"],
["override"]
)
merged_config = deepcopy(left_config)
merger.merge(merged_config, right_config)
return merged_config
def merge(base: config_dict,
overrides: config_dict,
allow_in_place: bool = True) -> config_dict:
merger = Merger(
[(dict, ["merge"])], # strategies to use for each type
["override"], # fallback strategies applied to all other types
["override"] # strategies when the types conflict
)
if not allow_in_place:
base = deepcopy(base)
overrides = deepcopy(overrides)
merger.merge(base, overrides)
return base
def merger(ref_path, overriding_jcasc_yaml):
# load the jcasc defaults bundles int he docker image
with open(f'{ref_path}/defaults.yaml') as file:
default_jcasc = yaml.load(file, Loader=yaml.FullLoader)
# load the user supplied jcasc
with open(overriding_jcasc_yaml) as file:
overriding_jcasc = yaml.load(file, Loader=yaml.FullLoader)
# set up the merger with the merge stratigies
jcasc_merger = Merger(
[(list, ["override"]), (dict, [jcasc_dict_merge]), (set, ["union"])],
# fallback strategy
["override"],
# conflict strategy
["override"])
merged = jcasc_merger.merge(default_jcasc, overriding_jcasc)
merged_jcasc = f'{ref_path}/jenkins.yaml'
# write our merged jcasc yaml
write_yaml_file(merged_jcasc, merged)
return merged_jcasc
def parse_consul(value):
consul_config = json.loads(os.environ['CONSUL_CONFIG']) if 'CONSUL_CONFIG' in os.environ else {}
scheme = consul_config['scheme'] if 'scheme' in consul_config else 'http'
host = consul_config['host'] if 'host' in consul_config else '127.0.0.1'
port = consul_config['port'] if 'port' in consul_config else 8500
token = consul_config['token'] if 'token' in consul_config else None
consul_merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'], ['override'])
try:
consul = Consul(scheme=scheme, host=host, port=port, token=token)
_, kv_entries = consul.kv.get(recurse=True, key=value)
except ACLPermissionDenied:
print(BRIGHT_RED + 'Access denied connecting to: {}://{}:{}'.format(scheme, host, port) + RESET_ALL)
return ERROR
if not kv_entries:
# Mark as failed if we can't find the consul key
return ERROR
consul_dict = {}
for entry in kv_entries:
subkeys = entry['Key'].split('/')
value = entry['Value'].decode('utf-8') if hasattr(entry['Value'], 'decode') else entry['Value']
value = '' if value is None else value
if '/' in entry['Key']:
key = '{"' + entry['Key'].replace('/', '":{"') + '": "' + value + '"}'.ljust(len(subkeys)+1, '}')
consul_dict = consul_merger.merge(consul_dict, json.loads(key))
else:
consul_dict[entry['Key']] = value
# return subkeys relative to rootkey
rootkey = list(consul_dict.keys())[0]
return consul_dict[rootkey]
def merge_dicts(base: dict, other: dict, list_semantics='set_merge'):
'''
merges copies of the given dict instances and returns the merge result.
The arguments remain unmodified. However, it must be possible to copy them
using `copy.deepcopy`.
Merging is done using the `deepmerge` module. In case of merge conflicts, values from
`other` overwrite values from `base`.
By default, different from the original implementation, a "set-merge" will be applied to
lists. This results in deduplication and potential change of element order, which may be
undesired. In this case, set `list_semantics` to 'None'
'''
ensure_not_none(base)
ensure_not_none(other)
from deepmerge import Merger
if list_semantics == 'set_merge':
# monkey-patch merge-strategy for lists
list_merge_strategy = Merger.PROVIDED_TYPE_STRATEGIES[list]
list_merge_strategy.strategy_merge = lambda c, p, base, other: list(
set(base) | set(other))
strategy_cfg = [(list, ['merge']), (dict, ['merge'])]
merger = Merger(strategy_cfg, ['override'], ['override'])
from copy import deepcopy
# copy dicts, so they remain unmodified
return merger.merge(deepcopy(base), deepcopy(other))
def load_config(config_file: Union[str, Path]) -> Dict:
"""
Loads and validates YAML config for this exporter and fills in default values
:param config_file:
:return: config as dictionary
"""
yaml = YAML()
with open(config_file) as f:
config = yaml.load(f)
# merge with default config: use 'override' for lists to let users replace extractor setting entirely
merger = Merger([(list, "override"), (dict, "merge")], ["override"],
["override"])
config = merger.merge(DEFAULT_CONFIG, config)
for param in [IP_ADDRESS, PASSWORD]:
if not param in config:
raise ValueError(
f"'{param}' is a mandatory config parameter, but it is missing in the YAML configuration file. Please see README.md for an example."
)
if config[EXPORTER][TIMEOUT_SECONDS] <= 0:
raise ValueError(f"'{TIMEOUT_SECONDS} must be positive.")
if config[EXPORTER][PORT] < 0 or config[EXPORTER][PORT] > 65535:
raise ValueError(f"Invalid exporter port.")
if not config[EXPORTER][EXTRACTORS]:
raise ValueError(
"The config file needs to specify at least one family of metrics.")
config[EXPORTER][EXTRACTORS] = sorted(set(config[EXPORTER][EXTRACTORS]))
return config
def merge_value(reference, new_value):
merger = Merger([(list, ["append"]), (dict, ["merge"])], ["override"], ["override"])
if isinstance(new_value, (list, set, dict)):
new_reference = merger.merge(reference, new_value)
else:
raise TypeError("Cannot handle merge_value of type {}".format(type(new_value)))
return new_reference
def get_section(section: str) -> Dict[str, Any]:
"""
Load the section config from config-default.toml and config.toml.
Use deepmerge.Merger to merge the configuration from config.toml
and override that of config-default.toml.
"""
# Load default configuration
if not Path("config-default.toml").exists():
raise MetricityConfigurationError("config-default.toml is missing")
with open("config-default.toml", "r") as default_config_file:
default_config = toml.load(default_config_file)
# Load user configuration
user_config = {}
if Path("config.toml").exists():
with open("config.toml", "r") as default_config_file:
user_config = toml.load(default_config_file)
# Merge the configuration
merger = Merger([(dict, "merge")], ["override"], ["override"])
conf = merger.merge(default_config, user_config)
# Check whether we are missing the requested section
if not conf.get(section):
raise MetricityConfigurationError(
f"Config is missing section '{section}'")
return conf[section]
def main():
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
doc = {}
doc['openapi'] = '3.0.1'
doc['info'] = {
'title': 'NBA Stats API',
'description':
'OpenAPI documentation for the NBA Stats API generated by sigmer/nba-stats-openapi',
'version': __version__
}
doc['servers'] = [{'url': base_url + base_path}]
for endpoint in endpoints:
handle_endpoint(doc, endpoint)
# Uncomment to include header parameters
# for key, value in headers.items():
# components = doc['components']
# parameters = components['parameters']
# header = {
# 'name': key,
# 'in': 'header',
# 'required': True,
# 'schema': {'type': 'string'},
# 'example': value
# }
# parameters[key] = header
# components['parameters'] = parameters
# doc['components'] = components
default = {}
with open('/'.join([root_dir, 'templates', 'default.yaml'])) as f:
default = yaml.load(f, Loader=yaml.Loader)
merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'],
['override'])
merger.merge(doc, default)
dist_dir = '/'.join([root_dir, 'dist/'])
os.makedirs(os.path.dirname(dist_dir), exist_ok=True)
with open(dist_dir + 'swagger.yaml', 'w') as output:
yaml.dump(doc, output, sort_keys=False)
def _update_cfg_with_base(cfg):
if cfg["__BASE__"]:
base_path = cfg.pop("__BASE__")
if not os.path.exists(base_path):
# __file__: quant_pack/apis/env.py
project_root = Path(__file__).absolute().parents[2]
base_path = os.path.join(project_root, base_path)
base_cfg = yaml.load(open(base_path, "r", encoding="utf-8"),
Loader=yaml.SafeLoader)
base_cfg = _update_cfg_with_base(base_cfg)
# strategies for merging configs:
# - override lists in BASE by the counterparts in current config, such that
# each param-group can get the latest, non-duplicated config;
# - merge dicts recursively, such that derived config can be written as concisely as possible;
conf_merger = Merger([(list, "override"), (dict, "merge")],
["override"], ["override"])
conf_merger.merge(base_cfg, cfg)
cfg = base_cfg
return cfg
class GraphGenerator(object):
"""Convert a list of key-value luples into a graph-like hash.
e.g [ ('key1', 'value1'), ('key2', value2') ]
State is persisted to disk in JSON format
"""
def __init__(self, filename='graph.json'):
self.filename = filename
self.dict_merger = Merger([(list, ["append"]), (dict, ["merge"])],
["override"], ["override"])
self.load()
def __del__(self):
self.persist()
def load(self):
'''Load graph from file'''
try:
with open(self.filename) as f:
self.graph = json.load(f)
except IOError:
self.graph = {}
def persist(self):
''' Persist graph to file'''
with open('{}.new'.format(self.filename), 'w') as json_output:
json.dump(self.graph, json_output)
os.rename('{}.new'.format(self.filename), self.filename)
def add_list(self, list):
""" list = [ ('key1', 'value1'), ('key2', 'value2')...('keyN', 'valueN') ] """
new_graph = self._parse_list(list)
self.dict_merger.merge(self.graph, new_graph)
def _parse_list(self, list):
try:
key, value = list.pop(0)
return self._generate_node(key, value, self._parse_list(list))
except IndexError:
return {}
def _generate_node(self, key, value, edges):
return {key: {'value': value, '_edges': edges}}
def merge(*dicts):
result_dict = {}
merger = Merger(
# merge strategies - for each type.
[
(dict, ['merge']),
(list, ['override']),
],
# fallback strategies - for all other types
['override'],
# conflict strategies
['override'],
)
for next_dict in dicts:
merger.merge(result_dict, next_dict)
return result_dict
def merge_dicts(base: dict, *other: dict, list_semantics='merge'):
'''
merges copies of the given dict instances and returns the merge result.
The arguments remain unmodified. However, it must be possible to copy them
using `copy.deepcopy`.
Merging is done using the `deepmerge` module. In case of merge conflicts, values from
`other` overwrite values from `base`.
By default, different from the original implementation, a merge will be applied to
lists. This results in deduplication retaining element order. The elements from `other` are
appended to those from `base`.
'''
not_none(base)
not_empty(other)
from deepmerge import Merger
if list_semantics == 'merge':
# monkey-patch merge-strategy for lists
list_merge_strategy = Merger.PROVIDED_TYPE_STRATEGIES[list]
list_merge_strategy.strategy_merge = lambda c, p, base, other: \
list(base) + [e for e in other if e not in base]
strategy_cfg = [(list, ['merge']), (dict, ['merge'])]
merger = Merger(strategy_cfg, ['override'], ['override'])
elif list_semantics is None:
strategy_cfg = [(dict, ['merge'])]
merger = Merger(strategy_cfg, ['override'], ['override'])
else:
raise NotImplementedError
from copy import deepcopy
return functools.reduce(
lambda b, o: merger.merge(b, deepcopy(o)),
[base, *other],
{},
)
courses.remove(cls.replace('!', ''))
courses += [i for i in bundle if not i.startswith('!')]
return classes
config = {}
config_files = [
'config_1',
'config_2',
'config_3',
'config_6'
]
for c in config_files:
merger.merge(config, yaml.load(open('configs/%s.yml' % (c,), 'r')))
config['tclasses'] = transform_classes(config)
config['blocks'] = config['blocks'] if 'blocks' in config else {}
scen = Scenario('Schedule', horizon=(DAYS * 5))
# Will be created when needed
teachers = {}
classes = {}
for cls, _ in config['tclasses'].items():
classes[cls] = scen.Resource(cls)
courses = {}
for course, info in config['courses'].items():
print('Parsing P-factor data...')
pfactor_data_dict = du.parse_pfactor_data(config['p-factor'], data_dict)
datasets.append(pfactor_data_dict)
print('Parsing precipitation data...')
precip_data_dict = du.parse_precipitation_data(config['precipitation_dir'],
data_dict,
int(config['timesteps']))
datasets.append(precip_data_dict)
# Merge the datasets
print('Merging datasets...')
my_merger = Merger([(list, ["override"]), (dict, ["merge"])], ["override"],
["override"])
for dataset in datasets:
data_dict = my_merger.merge(data_dict, dataset)
# Converting RiverReaches to EstuaryReaches, where applicable
print('Setting tidal bounds...')
data_dict = du.parse_tidal_bounds(config['tidal_bounds'], data_dict)
# Filling in the gaps
print('Filling the gaps...')
data_dict["dimensions"].update({"d": 2, "state": 7, "form": 4, "n": 5})
for grid_cell_ref, grid_cell in data_dict.items():
if grid_cell_ref not in [
"dimensions", "grid_dimensions[d]",
"routed_reaches[branches][seeds][river_reach_ref]"
]:
x = grid_cell['x']
y = grid_cell['y']
class BaseDBHandler(object):
"""Base handler for db."""
__version__ = 'v4'
db_defaults = load_config(__version__).db.connection.base
_df_class = 'base_df'
_config = AttrDict()
_df_name = 'base_df'
_columns = None
def __new__(cls,
db_class: str = None,
db_engine: str = None,
**kwargs) -> object:
"""Create object.
Args:
db_class (str): database class (e.g. 'meta')
db_engine (str): database engine (e.g. 'sqlite')
**kwargs: DB-handler specific arguments
Returns:
(object): the corresponding handler object
"""
if cls is BaseDBHandler:
handler = cls._get_handler(db_class, db_engine)
return super(BaseDBHandler, cls).__new__(handler)
else:
return super(BaseDBHandler, cls).__new__(cls)
@classmethod
def _get_handler(cls, db_class, db_engine=None):
"""Returns an appropriate handler.
Args:
db_class (str): database class (e.g. 'meta')
db_engine (str): database engine (e.g. 'tinydb')
Returns:
(handler): database handler object
"""
# Load default config
config = load_config(cls.__version__)
# Check if the db_class is available
if db_class not in DB_HANDLERS.keys():
raise ValueError('Unsupported db_class: {}'.format(db_class))
# Get db_engine from environment variable if not specified
if db_engine is None:
db_engine = os.environ.get(
'PYDTK_{}_DB_ENGINE'.format(db_class.upper()), None)
# Get the default engine if not specified
if db_engine is None:
try:
db_defaults = getattr(config.db.connection, db_class)
db_engine = db_defaults.engine
except (ValueError, AttributeError):
raise ValueError(
'Could not find the default value for `db_engine`')
# Check if the corresponding handler is registered
if db_engine not in DB_HANDLERS[db_class].keys():
raise ValueError('Unsupported db_engine: {}'.format(db_engine))
# Get a DB-handler supporting the engine
return DB_HANDLERS[db_class][db_engine]
def __init__(self,
db_engine=None,
db_host=None,
db_name=None,
db_username=None,
db_password=None,
df_name=None,
read_on_init=False,
**kwargs):
"""Initialize BaseDBHandler.
Args:
db_engine (str): database engine (if None, the one in the config file will be used)
db_host (str): database HOST (if None, the one in the config file will be used)
db_name (str): database name (if None, the one in the config file will be used)
db_username (str): username (if None, the one in the config file will be used)
db_password (str): password (if None, the one in the config file will be used)
df_name (str): dataframe (table in DB) name (if None, class default value will be used)
read_on_init (bool): if True, dataframe will be read from database on initialization
"""
super(BaseDBHandler, self).__init__()
self.logger = logging.getLogger(__name__)
self._cursor = 0
self._data = {}
self._uuids_to_remove = []
self._count_total = 0
if df_name is not None:
self.df_name = df_name
# Load config
config = load_config(self.__version__)
try:
self._config = ConfigDict(config['db']['df_class'][self._df_class])
except KeyError:
self._config = ConfigDict()
# Initialize deepmerger
self._merger = Merger(
# pass in a list of tuple, with the
# strategies you are looking to apply
# to each type.
[(dict, ['merge']),
(list, [_deepmerge_append_list_unique, 'append'])],
# next, choose the fallback strategies,
# applied to all other types:
["override"],
# finally, choose the strategies in
# the case where the types conflict:
["override"])
# Initialize database
self._initialize_engine(db_engine, db_host, db_name, db_username,
db_password)
self._load_config_from_db()
# Fetch table
if read_on_init:
self.read()
def __len__(self):
"""Return number of recoreds."""
return len(self.data)
def __iter__(self):
"""Return iterator."""
return self
def __next__(self):
"""Return the next item."""
if self._cursor >= len(self):
self._cursor = 0
raise StopIteration()
# Grab data
data = self[self._cursor]
# Increment
self._cursor += 1
return data
def __getitem__(self, idx, remove_internal_columns=True):
"""Return data at index idx.
Args:
idx (int): index of the target data
remove_internal_columns (bool): if True, internal columns are removed
Returns:
(dict): data
"""
data = self.data[idx]
# Delete internal column
if remove_internal_columns:
if '_uuid' in data.keys():
del data['_uuid']
if '_creation_time' in data.keys():
del data['_creation_time']
return data
def _initialize_engine(self,
db_engine=None,
db_host=None,
db_name=None,
db_username=None,
db_password=None):
"""Initialize DB engine.
Args:
db_engine (str): database engine (if None, the one in the config file will be used)
db_host (str): database HOST (if None, the one in the config file will be used)
db_name (str): database name (if None, the one in the config file will be used)
db_username (str): username (if None, the one in the config file will be used)
db_password (str): password (if None, the one in the config file will be used)
"""
if db_engine is None:
db_engine = self.db_defaults.engine
if db_host is None:
db_host = self.db_defaults.host
if db_username is None:
db_username = self.db_defaults.username
if db_password is None:
db_password = self.db_defaults.password
self._db_engine = db_engine
if db_engine in DB_ENGINES.keys():
self._db = DB_ENGINES[db_engine].connect(
db_host=db_host,
db_name=db_name,
db_username=db_username,
db_password=db_password,
collection_name=self._df_name,
handler=self,
)
self._config_db = DB_ENGINES[db_engine].connect(
db_host=db_host,
db_name=db_name,
db_username=db_username,
db_password=db_password,
collection_name='--config--{}'.format(self._df_name),
handler=self,
)
else:
raise ValueError("Unsupported engine: {}".format(db_engine))
def _load_config_from_db(self):
"""Load configs from DB."""
if self._db_engine not in DB_ENGINES.keys():
return
try:
candidates = DB_ENGINES[self._db_engine].read(self._config_db,
handler=self)
if len(candidates) > 0:
if isinstance(candidates[0][0], dict):
self._config = ConfigDict(candidates[0][0])
else:
raise TypeError('Unexpected type')
except Exception as e:
logging.warning('Failed to load configs from DB: {}'.format(
str(e)))
def _save_config_to_db(self):
"""Save configs to DB."""
if self._db_engine not in DB_ENGINES.keys():
return
try:
config = dict(self._config)
config.update({'_uuid': '__config__'})
config = [config]
DB_ENGINES[self._db_engine].upsert(self._config_db,
data=config,
handler=self)
except Exception as e:
logging.warning('Failed to save configs to DB: {}'.format(str(e)))
def _get_uuid_from_item(self, data_in):
"""Return UUID of the given item.
Args:
data_in (dict or pandas.Series): dict or Series containing data
Returns:
(str): UUID
"""
hash_target_columns = \
self._config['index_columns'] if 'index_columns' in self._config.keys() else []
item = data_in
if isinstance(item, pd.Series):
item = data_in.to_dict()
pre_hash = ''.join([
'{:.09f}'.format(item[column])
if isinstance(item[column], float) else str(item[column].keys())
if isinstance(item[column], dict) else str(item[column])
for column in hash_target_columns if column in item.keys()
])
pre_hash = pre_hash.encode('utf-8')
uuid = hashlib.md5(pre_hash).hexdigest()
return uuid
def _read(self, **kwargs):
if self._db_engine is None:
raise DatabaseNotInitializedError()
elif self._db_engine in DB_ENGINES.keys():
func = DB_ENGINES[self._db_engine].read
available_args = set(inspect.signature(func).parameters.keys())
unavailable_args = set([k for k, v in kwargs.items() if v is not None]) \
.difference(available_args)
if len(unavailable_args) > 0:
logging.warning(
'DB-engine "{0}" does not support args: {1}'.format(
self._db_engine, list(unavailable_args)))
return func(self._db, handler=self, **kwargs)
else:
raise ValueError('Unsupported DB engine: {}'.format(
self._db_engine))
def read(self,
df_name=None,
query=None,
pql=None,
where=None,
group_by=None,
order_by=None,
limit=None,
offset=None,
**kwargs):
"""Read data from SQL.
Args:
df_name (str): Dataframe name to read
query (str SQL query or SQLAlchemy Selectable): query to select items
pql (PQL): Python-Query-Language to select items
where (str): query string for filtering items
group_by (str): column name to group
order_by (srt): column name to sort by
limit (int): number of items to return per a page
offset (int): offset of cursor
**kwargs: kwargs for function `pandas.read_sql_query`
or `influxdb.DataFrameClient.query`
"""
if df_name is not None:
self.df_name = df_name
data, self._count_total = self._read(query=query,
pql=pql,
where=where,
group_by=group_by,
order_by=order_by,
limit=limit,
offset=offset,
**kwargs)
# Check if UUID exists in each value
for value in data:
if '_uuid' not in value.keys():
raise ValueError('"_uuid" not found in data')
self._data = {record['_uuid']: record for record in data}
def _upsert(self, data):
"""Upsert data to DB.
Args:
data (list): data to save
"""
if self._db_engine is None:
raise DatabaseNotInitializedError()
elif self._db_engine in DB_ENGINES.keys():
DB_ENGINES[self._db_engine].upsert(self._db, data)
else:
raise ValueError('Unsupported DB engine: {}'.format(
self._db_engine))
def save(self, **kwargs):
"""Save data to DB."""
self._remove(self._uuids_to_remove)
self._uuids_to_remove = []
self._upsert(list(self._data.values()))
self._save_config_to_db()
def _remove(self, uuids):
"""Remove data from DB.
Args:
uuids [str]: A list of unique IDs
"""
if self._db_engine is None:
raise DatabaseNotInitializedError()
elif self._db_engine in DB_ENGINES.keys():
DB_ENGINES[self._db_engine].remove(self._db, uuids)
else:
raise ValueError('Unsupported DB engine: {}'.format(
self._db_engine))
def _drop_table(self, name):
"""Drop table from DB.
Args:
name (str): Name of table (collection)
"""
if self._db_engine is None:
raise DatabaseNotInitializedError()
elif self._db_engine in DB_ENGINES.keys():
DB_ENGINES[self._db_engine].drop_table(self._db, name)
else:
raise ValueError('Unsupported DB engine: {}'.format(
self._db_engine))
def add_data(self, data_in, strategy='overwrite', **kwargs):
"""Add data to db.
Args:
data_in (dict): a dict containing data
strategy (str): 'merge' or 'overwrite'
"""
assert strategy in ['merge', 'overwrite'], 'Unknown strategy.'
data = deepcopy(data_in)
if '_uuid' not in data.keys() or '_creation_time' not in data.keys():
# Add _uuid and _creation_time
data['_uuid'] = self._get_uuid_from_item(data)
data['_creation_time'] = datetime.now().timestamp()
if data['_uuid'] in self._data.keys():
if strategy == 'merge':
base_data = self._data[data['_uuid']]
data = self._merger.merge(base_data, data)
# Add new columns (keys) to config
columns = self._config['columns'] if 'columns' in self._config.keys(
) else []
columns_existing = [c['name'] for c in columns]
columns_in_data = list(data_in.keys())
new_columns = []
for new_column in set(columns_in_data).difference(columns_existing):
name = new_column
dtype = type(data_in[new_column]).__name__
aggregation = 'first'
new_columns.append({
'name': name,
'dtype': dtype,
'aggregation': aggregation,
'display_name': name,
})
columns += new_columns
self._config['columns'] = columns
self._data.update({data['_uuid']: data})
def remove_data(self, data):
"""Remove data-record from DB.
Args:
data (dict): a dict containing the target data or '_uuid' in keys.
"""
if '_uuid' in data.keys():
uuid = data['_uuid']
else:
uuid = self._get_uuid_from_item(data)
# Remove from in-memory data
if uuid in self._data.keys():
del self._data[uuid]
# Remove from DB on saving
self._uuids_to_remove.append(uuid)
def _df_from_dicts(self, dicts):
"""Create a DataFrame from a list of dicts.
Args:
dicts (list): list of dicts
Returns:
(pd.DataFrame): a data-frame
"""
columns = self._config['columns'] if 'columns' in self._config.keys(
) else []
df = pd.concat(
[
pd.Series(name=c['name'],
dtype=dtype_string_to_dtype_object(c['dtype']))
for c in columns
if c['name'] != '_uuid' and c['name'] != '_creation_time'
] # noqa: E501
+ [
pd.Series(name='_uuid', dtype=str),
pd.Series(name='_creation_time', dtype=float)
],
axis=1)
df.set_index('_uuid', inplace=True)
df = pd.concat([df, pd.DataFrame.from_records(dicts)])
return df
def _to_display_names(self, df, inplace=False):
"""Rename columns to display-names.
Args:
df (pd.DataFrame): A data-frame
inplace (bool): If true, df will be modified in-place.
Returns:
(pd.DataFrame): A data-frame with renamed columns if inplace=false, otherwise None.
"""
column_renames = {
c['name']: c['display_name']
for c in self._config['columns'] if isinstance(c, dict)
and 'name' in c.keys() and 'display_name' in c.keys()
}
return df.rename(columns=column_renames, inplace=inplace)
@property
def data(self):
"""Return data.
Returns:
(list): list of dicts
"""
return list(self._data.values())
@data.setter
def data(self, data):
"""Setter for self.data.
Args:
data (list): new-data
"""
assert isinstance(data, list)
for value in data:
self.add_data(value)
@property
def columns(self):
"""Return columns of DF."""
df = self._df_from_dicts(self.data)
return df.columns.tolist()
@property
def df(self):
"""Return df."""
df = self._df_from_dicts(self.data)
df = self._to_display_names(df)
return df
@property
def count_total(self):
"""Return total number of rows."""
return self._count_total
@property
def config(self):
"""Return config."""
return self._config
class EZGrow():
def __init__(self):
self.conf = dict()
self.merger = Merger([(list, ["append"]), (dict, ["merge"])],
["override"], ["override"])
for c in ['/etc/ezgrow/ezgrow.json', '/site/etc/ezgrow/ezgrow.json']:
try:
with open(c, 'r') as s:
self.merger.merge(self.conf, json.load(s))
except Exception as e:
print("Can't load config %s: [%s]" % (c, e))
if len(self.conf) == 0:
raise RuntimeError('Config is empty, NOT rebooting the host')
def reload_sensors(self):
value = dict()
for f in self.conf['json']:
try:
with open(f) as s:
self.merger.merge(value, json.load(s))
except Exception as e:
print("Can't load sensor %s: [%s]" % (f, e))
return value
def update_watchdog(self):
try:
self.watchdog.write('X')
self.watchdog.flush()
except AttributeError:
print("Opening watchdog device %s" % self.conf['watchdog'])
self.watchdog = open(self.conf['watchdog'], 'w')
def reload_gpio(self):
status, gpio = getstatusoutput('gpio allreadall')
assert (not status), 'Failed to run gpio'
value = dict()
for line in filter(lambda x: len(x) > 10 and x[1][0].isdigit(),
map(lambda x: x.split(None), gpio.splitlines())):
value[int(line[1])] = 1 if line[5] == 'High' else 0 # col 1
value[int(line[8])] = 1 if line[12] == 'High' else 0 # col 2
return value
def reload_snmp(self):
on_value = self.conf['snmp']['value']['on']
return dict(map(lambda x: (x[0], on_value == self.snmp_get(x[1])), \
self.conf['snmp']['oid'].items()))
def snmp_get(self, oid):
status,snmp = getstatusoutput('snmpget -v1 -Oq -c%s %s %s' % \
(self.conf['snmp']['pass'], self.conf['snmp']['addr'], oid)
)
assert (not status), 'Failed to run snmpget'
for line in map(lambda x: x.split(None), snmp.splitlines()):
return line[1]
def snmp_set(self, oid, kind, value):
status,snmp = getstatusoutput('snmpset -v1 -Oq -c%s %s %s %s %s' % \
(self.conf['snmp']['pass'], self.conf['snmp']['addr'], oid, kind, value)
)
assert (not status), 'Failed to run snmpset'
return self.snmp_get(oid)
def avg(self): # return [min, max, avg]
value, count = [float('inf'), float('-inf'), 0.0], 0.0
for inside in self.conf['inside-temperature']:
for sensor in filter(lambda x: x.startswith(inside),
self.conf['tempdata']):
value[0] = min(
value[0],
float(self.conf['tempdata'][sensor]['temperature']))
value[1] = max(
value[1],
float(self.conf['tempdata'][sensor]['temperature']))
value[2] += float(self.conf['tempdata'][sensor]['temperature'])
count += 1.0
value[2] = value[2] / count if count != 0 else float('inf')
return value
def _any_low(self, name): # returns True if any pin is low
for pin in self.conf['gpio'][name]:
#print('Checking pin (l): %d' % pin)
if self.conf['gpiodata'][pin] == 0:
return True
return False
def _any_high(self, name): # returns True if any pin is high
for pin in self.conf['gpio'][name]:
#print('Checking pin (h): %d' % pin)
if self.conf['gpiodata'][pin] != 0:
return True
return False
def get_pump(self):
return not(self._any_low('leak') \
or self._any_high('water-high'))
def get_lamp(self):
tm = datetime.strptime(self.conf['timestamp'].strftime('%H:%M'),
'%H:%M')
t = self.conf['time'][self.conf['time']
['mode']] # { 'on': ..., 'off': ... }
return not ((tm > datetime.strptime(t['off'], '%H:%M')) \
and (tm < datetime.strptime(t['on'], '%H:%M')))
def get_ozone(self):
# TODO use holiday module and make sure don't turn on ozone
if self.conf['timestamp'].strftime('%a') in ['Sat', 'Sun']:
return False
h = int(self.conf['timestamp'].strftime('%H'))
m = int(self.conf['timestamp'].strftime('%M'))
# 1000 to 1100 and 10 minutes every hour until 1800
if h == 10 or (h > 8 and h < 19 and m < 15):
return True
return False
def get_fan_ext(self):
avg = self.avg()[1] # get maximum value
if avg in [float('inf'), float('-inf')]:
return True # safety net; turn on if no sensors
fan = self.conf['snmpdata']['fan-ext']
if fan:
# fan is on now
if avg < self.conf['temp']['off']:
return False
else:
# fan is off now
if avg > self.conf['temp']['on']:
return True
return fan != 0 # keep fan unchanged
def run(self):
for group in self.conf['gpio']:
for pin in self.conf['gpio'][group]:
status, gpio = getstatusoutput('gpio -g mode %d in' % pin)
assert (not status), 'Failed to run gpio mode in'
status, gpio = getstatusoutput('gpio -g mode %d down' % pin)
assert (not status), 'Failed to run gpio mode down'
while True:
# used for system schedulig (lamps, etc.)
self.conf['timestamp'] = datetime.now()
self.update_watchdog()
self.conf['gpiodata'] = self.reload_gpio()
self.update_watchdog()
self.conf['tempdata'] = self.reload_sensors()
self.update_watchdog()
self.conf['snmpdata'] = self.reload_snmp()
self.update_watchdog()
self.conf['update'] = {
'lamp': self.get_lamp(), # this only depend on timer
'pump': self.get_pump(), # this only depend on sensors
'fan-ext': self.get_fan_ext(),
'fan-int': True, # TODO some logic
'ozone': self.get_ozone(),
}
print(self.conf['update'])
for outlet in self.conf['update'].items():
self.update_watchdog()
if outlet[1] == self.conf['snmpdata'][outlet[0]]:
continue
oid = self.conf['snmp']['oid'][outlet[0]]
value = self.conf['snmp']['value'][
'on' if outlet[1] else 'off']
self.snmp_set(oid, 'i', value)
tm = datetime.now() # update timestamp
self.conf['timestamp'] = tm # '%s' % tm # is not JSON serializable
with open(tempfile.gettempdir() + '/status.yaml', 'w') as s:
pyaml.dump(self.conf, s) # , indent=4, sort_keys=True)
time.sleep(self.conf['sleep'] - tm.second % self.conf['sleep'])
