async def create_image(self, name, path):
img_id = None
path = Path(path).resolve()
with subprocess.Popen(tar_image_cmd(path),
stdout=subprocess.PIPE) as proc:
img_params = Prodict.from_dict({
'fileobj': proc.stdout,
'encoding': 'identity',
'tag': name,
'labels': def_labels(),
'stream': True
})
logger.info(f"building image {img_params} from {path}")
async for chunk in await self.dc.images.build(**img_params):
if isinstance(chunk, dict):
logger.debug(chunk)
if 'aux' in chunk:
img_id = underdict(chunk['aux'])
else:
logger.debug('chunk: %s %s', type(chunk), chunk)
logger.info('image created %s', img_id)
img = await self.dc.images.get(name)
return Prodict.from_dict(underdict(img))
async def store_stock(self, ctx, store_name: str, item_name: str):
store = Store.find_one({'name': store_name})
if not store:
return await ctx.send(
f'```fix\nCould not find store "{store_name}"\n```')
item = Item.find_one({'name': item_name})
if not item:
return await ctx.send(
f'```fix\nCould not find item "{item_name}"\n```')
store = Prodict.from_dict(store)
item = Prodict.from_dict(item)
if store.item_list:
if item.id in store['item_list']:
return await ctx.send(
f'```fix\nItem "{item_name}" already exists in "{store_name}"\n```'
)
store.item_list.append(item.id)
print('exist')
else:
print('not exist')
store.item_list = [item.id]
Store.save(store)
await ctx.send(f'```css\nStocked "{item_name}" in "{store_name}"\n```')
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
self._next_transition = model.job.next_transition
issues = []
with _open(self.cfg_stream) as stream:
users_list: List[dict] = yaml.safe_load(stream) or []
default_db_name = {db_uid: db.db_name for db_uid, db in model.aws.databases.items()
if DbStatus[db.status] >= DbStatus.ENABLED}
enabled_databases = list(default_db_name.keys())
users = {}
databases = {}
for user in users_list:
login = user["login"]
default_grant_type = user.get("default_grant_type", DEFAULT_GRANT_TYPE)
try:
permissions = self._parse_permissions(
user.get("permissions", []),
model.aws.single_region,
default_grant_type,
enabled_databases,
default_db_name)
users[login] = {
"db_username": login[:MAX_DB_USERNAME_LENGTH],
"permissions": permissions
}
for db_uid, grant_type in permissions.items():
databases.setdefault(db_uid, {"permissions": {}})["permissions"][login] = grant_type
except ValueError as e:
issues.append(Issue(level=IssueLevel.ERROR, type='USER', id=login, message=str(e)))
updates = Prodict(okta={"users": users}, aws={"databases": databases})
if self._next_transition:
updates.job = dict(next_transition=self._next_transition)
return updates, issues
def test_dict_reserved_keys():
with tc.assertRaises(TypeError):
Prodict(pop=5)
pd2 = Prodict(pop1=5)
with tc.assertRaises(TypeError):
pd2.pop = 5
async def view(self, ctx, store=None):
""" View store """
user = await author.get(ctx.author)
store_list = ''
stores = StoreModel.find()
loaded_store = None
if store is None:
if not stores:
return await ctx.send(f'```fix\nThere are no stores setup"\n```')
if stores.count() == 1:
loaded_store = Prodict.from_dict(stores[0])
item_list = ''
loaded_store.inventory = sorted(loaded_store.inventory, key=curried.get_in(['price']), reverse=False)
for i, item in enumerate(loaded_store.inventory):
item = Prodict.from_dict(item)
if user['quote_to'] != 'USD':
rates = await coins.rate_convert()
item.price = rates[user['quote_to']] * item.price
item.payout = rates[user['quote_to']] * item.payout
formatted_price = currency.symbol(user["quote_to"]) + '{0:.2f}'.format(item.price)
formatted_payout = "{0:.2f}".format(item.payout)
item_list += f'{i + 1}. {item.name}{" " * (15 - len(item.name))}{formatted_price}{" " * (10 - len(formatted_price))}{item.about.format(payout=formatted_payout, rate=str(item.rate) + " minutes")}\n'
return await ctx.send(
f'```py\n{user.quote_to}\nStore name: {loaded_store.name}\n\nItem{" " * (18 - len("item"))}Price{" " * (10 - len("price"))}Description\n\n{item_list}\n\nQoins represent a USD value by default, the prices will convert depending upon what quote currency you have set on your account. Use the "{self.config.prefix[0]}sq <currency symbol>" command to change it```')
for i, _store in enumerate(stores):
_store = Prodict.from_dict(_store)
item_count = len(_store.inventory)
store_list += f'\n{i + 1}. {_store.name}{" " * (12 - len("Name"))}{item_count}{" " * (10 - len(str(item_count)))}{_store.about}'
store_list_head = f'\nName{" " * (15 - len("Name"))}Item Count{" " * (10 - len("item"))}Description'
await ctx.send(f'```diff\nStore list:\n{store_list_head}{store_list}\n```')
return await ctx.send_help('view')
store_list = []
for i, _store in enumerate(stores):
store_list.append(_store)
if store.isnumeric():
if int(store) - 1 in range(len(store_list)):
loaded_store = store_list[int(store) - 1]
else:
loaded_store = StoreModel.find_one({'name': store})
if not loaded_store:
return await ctx.send(f'```fix\nCould not find store "{store}"\n```')
loaded_store = Prodict.from_dict(loaded_store)
item_list = ''
for i, item in enumerate(loaded_store.inventory):
item = Prodict.from_dict(item)
if user['quote_to'] != 'USD':
rates = await coins.rate_convert()
item.price = rates[user['quote_to']] * item.price
item.payout = rates[user['quote_to']] * item.payout
formatted_price = currency.symbol(user["quote_to"]) + '{0:.2f}'.format(item.price)
formatted_payout = "{0:.2f}".format(item.payout)
item_list += f'{i + 1}. {item.name}{" " * (18 - len(item.name))}{formatted_price}{" " * (10 - len(formatted_price))}{item.about.format(payout=formatted_payout)}\n'
await ctx.send(
f'```py\n{user.quote_to}\n{loaded_store.name}\n\nItem{" " * (21 - len("item"))}Price{" " * (10 - len("price"))}Description\n\n{item_list}```')
def __init__(self, images_path, images, container_params, container_env,
**kwargs):
self.dc = aiodocker.Docker()
self.initial_ports = list(range(8900, 8999))
self.available_ports = list(self.initial_ports)
self.images_path = images_path
self.images = Prodict.from_dict(images)
self.container_env = Prodict.from_dict(container_env)
self.container_params = Prodict.from_dict(container_params)
async def chech(p, params):
asyncio.sleep(random()*1)
check = Prodict(success=0, region=p.cityRu)
check.update({k: p[k] for k in copyattrs})
try:
state.myiter()
async with aiohttp.ClientSession() as chs:
proxy_auth = aiohttp.BasicAuth(p.user, p.password)
hostport = f"http://{p.host.strip()}:{p.port}"
async with chs.get(test_url, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as pr:
check.responseCode = pr.status
if pr.status == 200:
check.success = 1
check.contentSize = len(await pr.text())
async with chs.get(check_ip, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as ip_r:
if ip_r.status == 200:
check.extIp = (await ip_r.text()).strip()
state.succ()
except ClientConnectorError:
logger.warn('connection error: %s:%s', p.host, p.port)
except asyncio.TimeoutError:
logger.warn('asyncio timeout')
except TimeoutError:
logger.warn('timeout')
except Exception:
logger.exception('check err')
await asyncio.sleep(1)
try:
async with aiohttp.ClientSession() as ss:
async with ss.post(params.notify, json=check, timeout=10) as wh_r:
if wh_r.status != 200:
logger.error('webhook failed')
except Exception:
logger.exception('sending webhook err')
def __init__(self,
in_channels: int,
out_channels: int,
kernel_size: int = 1,
padding: int = 0,
stride: int = 1,
num_groups: int = 1,
norm: str = "GN",
gate_activation: str = "ReTanH",
gate_activation_kargs: dict = None):
super(DynamicBottleneck, self).__init__()
self.num_groups = num_groups
self.norm = norm
self.in_channels = in_channels
self.out_channels = out_channels
self.bottleneck = BasicBlock(in_channels,
out_channels,
stride=stride,
norm=norm,
activation=Prodict(NAME="ReLU",
INPLACE=True))
self.gate = SpatialGate(in_channels,
num_groups=num_groups,
kernel_size=kernel_size,
padding=padding,
stride=stride,
gate_activation=gate_activation,
gate_activation_kargs=gate_activation_kargs,
get_running_cost=self.get_running_cost)
self.init_parameters()
def completely_config(dir='.', conf='config.yaml',
env_fn='.env', env_local_fn='.env.local'):
root = Path(os.getcwd())
load_dotenv(dotenv_path=root/env_fn)
load_dotenv(dotenv_path=root/env_local_fn)
env = os.environ['ENV'] = os.environ.get('ENV', ENV_DEV)
name = os.environ['NAME'] = os.getenv('NAME', socket.gethostname())
print(__package__)
tmplenv = Environment(
loader=FileSystemLoader(str(root))
)
tmpl = tmplenv.get_template(conf)
data = tmpl.render(**os.environ)
data = yaml.load(data)
data.update({
# use pre configured or detected service name
'name': data.get('name', name),
'env': env
})
data.update(data.pop(data['name'], {}))
return Prodict.from_dict(data)
def run_struct(self, name, network, memory, bind_ip, host_ports,
auto_remove, etc_hosts, env, **kwargs):
return Prodict.from_dict({
'Image': self.image.id,
'Hostname': name,
'Cmd': self.image.cmd,
'Labels': {
'inband': 'native'
},
'Env': [f"{k}={v}" for k, v in env.items()],
'StopSignal': 'SIGTERM',
'HostConfig': {
# 'AutoRemove': auto_remove,
'RestartPolicy': {
'Name': 'unless-stopped'
},
'PortBindings': {
p: [{
'HostIp': bind_ip,
'HostPort': str(hp)
}]
for hp, p in zip(host_ports, self.image.ports)
},
'ExtraHosts':
[f"{host}:{ip}" for host, ip in etc_hosts.items()],
'NetworkMode': network,
'Memory': memory
}
})
async def _buy(self, ctx, item_name: str):
""" Buy an item from store """
user = await author.get(ctx.author)
item = Item.find_one({'name': item_name})
if not item:
return await ctx.send(f'```fix\nCannot find item "{item_name}"\n```')
item = Prodict.from_dict(item)
if any(i['id'] == item.id for i in user.item_list):
return await ctx.send(f'```fix\nYou already own item "{item_name}"\n```')
if user['quote_to'] != 'USD':
rates = await coins.rate_convert()
item.price = rates[user['quote_to']] * item.price
item.payout = rates[user['quote_to']] * item.payout
if user.game.in_pocket < item.price:
return await ctx.send(f'```fix\nYou don\'t have enough money in your pocket\n```')
user.game.in_pocket = round(user.game.in_pocket - item.price)
user.item_list.append({
'id': item.id,
'last_run': datetime.now()
})
User.save(user)
await ctx.send(f'```css\nYou bought an item\n```')
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
with open(self.cfg_filename) as file:
rds_list: List[dict] = yaml.safe_load(file)
issues = []
databases = {}
for cfg_db in rds_list:
db_id = cfg_db["id"]
db_uid = f"{self.region}/{db_id}"
enabled = _to_bool(cfg_db.setdefault("enabled", True))
if enabled:
try:
master_password, password_age = self.pwd_resolver.resolve(db_id, cfg_db.get("master_password"))
db = {
"status": DbStatus.ENABLED.name,
"permissions": {},
"master_password": master_password,
"password_age": password_age,
}
except Exception as e:
issues.append(Issue(level=IssueLevel.ERROR, type="DB", id=db_uid, message=str(e)))
continue
else:
db = dict(status=DbStatus.DISABLED.name)
databases[db_uid] = db
return Prodict(aws={"databases": databases}), issues
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
issues = []
updates = {}
with open(self.cfg_filename) as file:
services = yaml.safe_load(file)
enabled_databases = [db_uid for db_uid, db in model.aws.databases.items()
if DbStatus[db.status] >= DbStatus.ENABLED]
for conn in services.get("glue_connections", []):
db_ref = conn['db']
db_id_list = wc_expand(db_ref, enabled_databases)
if not db_id_list:
issues.append(Issue(level=IssueLevel.ERROR, type='GLUE', id=db_ref,
message=f"Not existing and enabled DB instance reference '{db_ref}'"))
continue
pcr = conn.get("physical_connection_requirements", {})
supplied_db_names = conn.get("db_names")
if isinstance(supplied_db_names, str):
supplied_db_names = [supplied_db_names]
grant_type = conn.get("grant_type", DEFAULT_GRANT_TYPE)
for db_uid in db_id_list:
db = model.aws.databases[db_uid]
updates[db_uid] = {
"db_names": supplied_db_names or [db.db_name],
"grant_type": grant_type,
"physical_connection_requirements": {
"availability_zone": pcr.get("availability_zone", db.availability_zone),
"security_group_id_list": pcr.get("security_group_id_list", db.vpc_security_group_ids),
"subnet_id": pcr.get("subnet_id", db.primary_subnet),
},
}
return Prodict(aws={"glue_connections": updates}), issues
def __init__(self, cfg: List[Union[str, Path, dict]]):
self._instance = Prodict()
for item in cfg:
self.update(item)
self._ensure_has_all_params(Config.REQUIRED_PARAMS)
logger.debug(f"Configuration loaded from {cfg}")
async def run_container(self, name, params):
# build custom images
if False:
img_path = ''
else:
# rebuild base image
await self.create_image(self.images.base.name,
self.images.base.path)
# params for service image
img_path = self.images.collection.path
# service image
service_img_name = f'rst/service-{name}'
img = await self.create_image(service_img_name, img_path)
def take_port():
return {
'HostIp': self.container_params.bind_ip,
'HostPort': str(self.allocate_port())
}
ports = {
port: [take_port()]
for port in img.container_config.exposed_ports.keys() or {}
}
a_ports = [port[0]['HostPort'] for port in ports.values()]
env = {'NAME': name}
env.update(self.container_env)
config = Prodict.from_dict({
'Image': img.id,
'Hostname': name,
'Cmd': name,
'Ports': ports,
'Labels': def_labels(a_ports=a_ports),
'Env': [f"{k}={v}" for k, v in env.items()],
'StopSignal': 'SIGTERM',
'HostConfig': {
'RestartPolicy': {
'Name': 'unless-stopped'
},
'PortBindings': ports,
'NetworkMode': self.container_params.network,
'Memory': self.container_params.memory
}
})
print(config)
logger.info(f"starting container {name}. ports: {config.Ports}")
c = await self.dc.containers.create_or_replace(name, config)
await c.start()
await c.show()
c = inject_attrs(c)
logger.info(f'started container {c.attrs.name} [{c.attrs.id}]')
return short_info(c)
async def items(self, ctx):
""" View items """
items = Item.find()
item_list = ''
for item in items:
item = Prodict.from_dict(item)
item_list += f'{item.name} - {item.about}\nprice: {item.price} payout: {item.payout} rate: {item.rate}\n\n'
await ctx.send(f'```py\nItems:\n{item_list}```')
def _parse_input_config(cfg: Union[str, Path, dict]):
if isinstance(cfg, dict):
return Prodict.from_dict(cfg)
if isinstance(cfg, str) and "=" in cfg:
key, value = cfg.split("=")
parsed_value: Any
if value == "True":
parsed_value = True
elif value == "False":
parsed_value = False
elif re.match(r"^[-+]?[0-9]+$", value):
parsed_value = int(value)
elif re.match(r"^[-+]?[0-9]*\.?[0-9]+$", value):
parsed_value = float(value)
else:
parsed_value = value
key_chain = key.split(".")
result: Dict[str, Any] = {}
ref = result
while len(key_chain) > 1:
next_key = key_chain.pop(0)
ref[next_key] = {}
ref = ref[next_key]
ref[key_chain.pop(0)] = parsed_value
return Prodict.from_dict(result)
if isinstance(cfg, str):
cfg = Path(cfg)
if isinstance(cfg, Path):
if not cfg.suffix:
cfg = cfg.with_suffix(".json")
if not cfg.exists():
cfg = Config.CONFIG_DIR / cfg
if not cfg.exists():
raise FileNotFoundError(
f"Given config {str(cfg)} does not exist locally or as part of wtl."
)
json_data = json.load(Path(cfg).open())
return Prodict.from_dict(json_data)
raise ValueError(f"Unexpected config input: {cfg}")
def build_options_from_req(params: Dict):
"""
Build BuildOptions from request params
"""
return BuildOptions(nocache=req_to_bool(params.get('nocache', None)),
auto_remove=req_to_bool(params.get(
'auto_remove', None)),
env=pdict.from_dict(params.get('env', {})))
async def main(data, **params):
data = pdict.from_dict(data)
logger.debug('received message', **data)
if data.message:
await scheduler.spawn(handle_msg(data))
return {}
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
"""
Check if the users exist and retrieve their corresponding user_id and ssh_pubkey.
"""
okta = model.okta
session = async_retryable_session(self.executor)
futures = []
searcher = jmespath.compile(
"[*].[id, status, profile.sshPubKey] | [0]")
for login in okta.users:
future = session.get(
f"https://{okta.organization}.okta.com/api/v1/users?limit=1&search=profile.login+eq+"
+ urllib.parse.quote(f'"{login}"'),
headers=(self._http_headers()))
futures.append(future)
issues = []
users_ext = {}
logger.info(f"Checking Okta {okta.organization.capitalize()}'s Users:")
login_max_len = max(map(len, okta.users), default=0)
for login, future in zip(okta.users, futures):
result = future.result()
result.raise_for_status()
json_response = json.loads(result.content.decode())
match = searcher.search(json_response)
user_data = {}
if match:
user_id, status, ssh_pubkey = match
if status != "ACTIVE":
err_msg = f"status={status}"
elif ssh_pubkey:
err_msg = None
user_data = {
"user_id": user_id,
"ssh_pubkey": ssh_pubkey,
}
else:
status = "MISSING_SSH_PUBKEY"
err_msg = "Missing SSH PubKey"
else:
status = "ABSENT"
err_msg = "Not found in OKTA"
user_data["status"] = status
if err_msg:
color = "red"
issues.append(
Issue(level=IssueLevel.ERROR,
type="USER",
id=login,
message=err_msg))
else:
color = "green"
leader = "." * (2 + login_max_len - len(login))
logger.opt(colors=True).info(
f" {login} {leader} <{color}>{status}</{color}>")
users_ext[login] = user_data
return Prodict(okta={"users": users_ext}), issues
async def portfolio_value(user, coin_list, quote_to=None):
value = 0
if user['game']['portfolio']['coins']:
for coin in user['game']['portfolio']['coins']:
if quote_to is None:
quote_to = user["quote_to"]
coin = Prodict.from_dict(coin)
value += await get_value(coin.symbol, quote_to, coin.name, coin.amount, coin_list)
return value
async def inventory(self, ctx):
""" Check your item inventory """
user = await author.get(ctx.author)
mention = ctx.author.mention
item_list = ''
if user.inventory:
for item in user.inventory:
item = Prodict.from_dict(item)
item_list += f'\'{item.name}\'\n'
await ctx.send(f'```py\nInventory:\n{item_list}```{mention}')
def __init__(self, cfg: Iterable[Union[str, Path, dict]] = None):
cfg = cfg or []
self._instance = Prodict()
for item in cfg:
if item:
self.update(item)
self._ensure_has_all_params(Config.REQUIRED_PARAMS)
logger.debug(f"Configuration loaded from {cfg}")
async def enrich(ip, **params):
if state.geodata:
try:
location = state.geodata.get_location(ip, detailed=True)
if location and 'info' in location:
return handle_location(**pdict.from_dict(location['info']))
return response.error('Database not ready')
except Exception:
logger.exception('mmgeo error')
return response.error('Error while quering database')
def inject_attrs(cont):
attrs = underdict(cont._container)
attrs['name'] = (attrs['name']
if 'name' in attrs else attrs['names'][0]).strip('/')
attrs['short_id'] = attrs['id'][:12]
if 'state' in attrs and 'status' in attrs['state']:
attrs['status'] = attrs['state']['status']
if 'config' in attrs and 'labels' in attrs['config']:
attrs['labels'] = attrs['config']['labels']
cont.attrs = Prodict.from_dict(attrs)
return cont
def test_issue15(self):
"""url: https://github.com/ramazanpolat/prodict/issues/15
if the payload has a attribute named 'self' then we get a TypeError:
TypeError: __init__() got multiple values for argument 'self'
"""
try:
p = Prodict(self=1)
assert True
except TypeError:
assert False
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
"""Loads the optional `custom.yaml` from the configuration directory."""
issues = []
custom_yaml = f"{model.system.config_dir}/custom.yaml"
if os.path.exists(custom_yaml):
with open(custom_yaml) as file:
custom = yaml.safe_load(file)
# TODO: validate
else:
custom = {}
return Prodict(custom=custom), issues
def test_recursive_annotations3():
r = Recursive()
r.dynamic_dict_attr = {'a': 1, 'b': 2, 'c': 3}
r.dynamic_prodict_attr = Prodict.from_dict({'a': 1, 'b': 2, 'c': 3})
assert r.dynamic_dict_attr == {'a': 1, 'b': 2, 'c': 3}
assert type(r.dynamic_dict_attr) == Prodict
assert r.dynamic_prodict_attr == {'a': 1, 'b': 2, 'c': 3}
assert type(r.dynamic_prodict_attr) == Prodict
def generate(cls, count=100, skip_ratio=0.0):
"""
A loop to iterate over to generate data.
:param count: Loop count
:param skip_ratio: Ratio of skipping a loop, useful to mimic realistic data
:return: Returns a Prodict (dot-dict)
"""
for index in range(count):
# if random.randint(0, 100) <= skip_ratio * 100:
# continue
yield Prodict()
def test_recursive_annotations1():
r = Recursive()
assert r == {}
assert set(r.attr_names()) == {'prodict_key', 'simple_key'}
r.prodict_key = Prodict(a=1)
print('r.prodict_key =', r.prodict_key)
print('r.prodict_key.a =', r.prodict_key.a)
print('type(r.prodict_key) =', type(r.prodict_key))
assert r.prodict_key == {'a': 1}
assert r.prodict_key.a == 1
assert type(r.prodict_key) == Prodict