def test_dict_reserved_keys():
with tc.assertRaises(TypeError):
Prodict(pop=5)
pd2 = Prodict(pop1=5)
with tc.assertRaises(TypeError):
pd2.pop = 5
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
self._next_transition = model.job.next_transition
issues = []
with _open(self.cfg_stream) as stream:
users_list: List[dict] = yaml.safe_load(stream) or []
default_db_name = {db_uid: db.db_name for db_uid, db in model.aws.databases.items()
if DbStatus[db.status] >= DbStatus.ENABLED}
enabled_databases = list(default_db_name.keys())
users = {}
databases = {}
for user in users_list:
login = user["login"]
default_grant_type = user.get("default_grant_type", DEFAULT_GRANT_TYPE)
try:
permissions = self._parse_permissions(
user.get("permissions", []),
model.aws.single_region,
default_grant_type,
enabled_databases,
default_db_name)
users[login] = {
"db_username": login[:MAX_DB_USERNAME_LENGTH],
"permissions": permissions
}
for db_uid, grant_type in permissions.items():
databases.setdefault(db_uid, {"permissions": {}})["permissions"][login] = grant_type
except ValueError as e:
issues.append(Issue(level=IssueLevel.ERROR, type='USER', id=login, message=str(e)))
updates = Prodict(okta={"users": users}, aws={"databases": databases})
if self._next_transition:
updates.job = dict(next_transition=self._next_transition)
return updates, issues
async def enrich(ua, **params):
"""
Detect device type using User-Agent string
https://github.com/selwin/python-user-agents
"""
try:
parsed = parse(ua)
res = Prodict(
os_family=parsed.os.family,
os_version=list(v if isinstance(v, int)
else 0 for v in parsed.os.version),
browser_family=parsed.browser.family,
browser_version=list(v if isinstance(
v, int) else 0 for v in parsed.browser.version,),
device_family=parsed.device.family,
device_brand=parsed.device.brand,
device_model=parsed.device.model)
res.is_bot = int(parsed.is_bot)
res.is_tables = int(parsed.is_tablet)
res.is_mob = int(parsed.is_mobile)
res.is_pc = int(parsed.is_pc)
return res
except Exception:
logger.exception('handle ex')
return response.error()
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
issues = []
updates = {}
with open(self.cfg_filename) as file:
services = yaml.safe_load(file)
enabled_databases = [db_uid for db_uid, db in model.aws.databases.items()
if DbStatus[db.status] >= DbStatus.ENABLED]
for conn in services.get("glue_connections", []):
db_ref = conn['db']
db_id_list = wc_expand(db_ref, enabled_databases)
if not db_id_list:
issues.append(Issue(level=IssueLevel.ERROR, type='GLUE', id=db_ref,
message=f"Not existing and enabled DB instance reference '{db_ref}'"))
continue
pcr = conn.get("physical_connection_requirements", {})
supplied_db_names = conn.get("db_names")
if isinstance(supplied_db_names, str):
supplied_db_names = [supplied_db_names]
grant_type = conn.get("grant_type", DEFAULT_GRANT_TYPE)
for db_uid in db_id_list:
db = model.aws.databases[db_uid]
updates[db_uid] = {
"db_names": supplied_db_names or [db.db_name],
"grant_type": grant_type,
"physical_connection_requirements": {
"availability_zone": pcr.get("availability_zone", db.availability_zone),
"security_group_id_list": pcr.get("security_group_id_list", db.vpc_security_group_ids),
"subnet_id": pcr.get("subnet_id", db.primary_subnet),
},
}
return Prodict(aws={"glue_connections": updates}), issues
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
with open(self.cfg_filename) as file:
rds_list: List[dict] = yaml.safe_load(file)
issues = []
databases = {}
for cfg_db in rds_list:
db_id = cfg_db["id"]
db_uid = f"{self.region}/{db_id}"
enabled = _to_bool(cfg_db.setdefault("enabled", True))
if enabled:
try:
master_password, password_age = self.pwd_resolver.resolve(db_id, cfg_db.get("master_password"))
db = {
"status": DbStatus.ENABLED.name,
"permissions": {},
"master_password": master_password,
"password_age": password_age,
}
except Exception as e:
issues.append(Issue(level=IssueLevel.ERROR, type="DB", id=db_uid, message=str(e)))
continue
else:
db = dict(status=DbStatus.DISABLED.name)
databases[db_uid] = db
return Prodict(aws={"databases": databases}), issues
def __init__(self, cfg: List[Union[str, Path, dict]]):
self._instance = Prodict()
for item in cfg:
self.update(item)
self._ensure_has_all_params(Config.REQUIRED_PARAMS)
logger.debug(f"Configuration loaded from {cfg}")
def __init__(self,
in_channels: int,
out_channels: int,
kernel_size: int = 1,
padding: int = 0,
stride: int = 1,
num_groups: int = 1,
norm: str = "GN",
gate_activation: str = "ReTanH",
gate_activation_kargs: dict = None):
super(DynamicBottleneck, self).__init__()
self.num_groups = num_groups
self.norm = norm
self.in_channels = in_channels
self.out_channels = out_channels
self.bottleneck = BasicBlock(in_channels,
out_channels,
stride=stride,
norm=norm,
activation=Prodict(NAME="ReLU",
INPLACE=True))
self.gate = SpatialGate(in_channels,
num_groups=num_groups,
kernel_size=kernel_size,
padding=padding,
stride=stride,
gate_activation=gate_activation,
gate_activation_kargs=gate_activation_kargs,
get_running_cost=self.get_running_cost)
self.init_parameters()
async def chech(p, params):
asyncio.sleep(random()*1)
check = Prodict(success=0, region=p.cityRu)
check.update({k: p[k] for k in copyattrs})
try:
state.myiter()
async with aiohttp.ClientSession() as chs:
proxy_auth = aiohttp.BasicAuth(p.user, p.password)
hostport = f"http://{p.host.strip()}:{p.port}"
async with chs.get(test_url, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as pr:
check.responseCode = pr.status
if pr.status == 200:
check.success = 1
check.contentSize = len(await pr.text())
async with chs.get(check_ip, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as ip_r:
if ip_r.status == 200:
check.extIp = (await ip_r.text()).strip()
state.succ()
except ClientConnectorError:
logger.warn('connection error: %s:%s', p.host, p.port)
except asyncio.TimeoutError:
logger.warn('asyncio timeout')
except TimeoutError:
logger.warn('timeout')
except Exception:
logger.exception('check err')
await asyncio.sleep(1)
try:
async with aiohttp.ClientSession() as ss:
async with ss.post(params.notify, json=check, timeout=10) as wh_r:
if wh_r.status != 200:
logger.error('webhook failed')
except Exception:
logger.exception('sending webhook err')
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
"""
Check if the users exist and retrieve their corresponding user_id and ssh_pubkey.
"""
okta = model.okta
session = async_retryable_session(self.executor)
futures = []
searcher = jmespath.compile(
"[*].[id, status, profile.sshPubKey] | [0]")
for login in okta.users:
future = session.get(
f"https://{okta.organization}.okta.com/api/v1/users?limit=1&search=profile.login+eq+"
+ urllib.parse.quote(f'"{login}"'),
headers=(self._http_headers()))
futures.append(future)
issues = []
users_ext = {}
logger.info(f"Checking Okta {okta.organization.capitalize()}'s Users:")
login_max_len = max(map(len, okta.users), default=0)
for login, future in zip(okta.users, futures):
result = future.result()
result.raise_for_status()
json_response = json.loads(result.content.decode())
match = searcher.search(json_response)
user_data = {}
if match:
user_id, status, ssh_pubkey = match
if status != "ACTIVE":
err_msg = f"status={status}"
elif ssh_pubkey:
err_msg = None
user_data = {
"user_id": user_id,
"ssh_pubkey": ssh_pubkey,
}
else:
status = "MISSING_SSH_PUBKEY"
err_msg = "Missing SSH PubKey"
else:
status = "ABSENT"
err_msg = "Not found in OKTA"
user_data["status"] = status
if err_msg:
color = "red"
issues.append(
Issue(level=IssueLevel.ERROR,
type="USER",
id=login,
message=err_msg))
else:
color = "green"
leader = "." * (2 + login_max_len - len(login))
logger.opt(colors=True).info(
f" {login} {leader} <{color}>{status}</{color}>")
users_ext[login] = user_data
return Prodict(okta={"users": users_ext}), issues
def __init__(self, cfg: Iterable[Union[str, Path, dict]] = None):
cfg = cfg or []
self._instance = Prodict()
for item in cfg:
if item:
self.update(item)
self._ensure_has_all_params(Config.REQUIRED_PARAMS)
logger.debug(f"Configuration loaded from {cfg}")
def test_issue15(self):
"""url: https://github.com/ramazanpolat/prodict/issues/15
if the payload has a attribute named 'self' then we get a TypeError:
TypeError: __init__() got multiple values for argument 'self'
"""
try:
p = Prodict(self=1)
assert True
except TypeError:
assert False
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
"""Loads the optional `custom.yaml` from the configuration directory."""
issues = []
custom_yaml = f"{model.system.config_dir}/custom.yaml"
if os.path.exists(custom_yaml):
with open(custom_yaml) as file:
custom = yaml.safe_load(file)
# TODO: validate
else:
custom = {}
return Prodict(custom=custom), issues
def test_pickle(self):
try:
encoded = pickle.dumps(Prodict(a=42))
decoded = pickle.loads(encoded)
assert decoded.a == 42
# p = Prodict(a=1, b=2)
# encoded = pickle.dumps(p)
# print(encoded)
# decoded = pickle.loads(encoded)
# print(decoded)
except:
assert False
def test_recursive_annotations1():
r = Recursive()
assert r == {}
assert set(r.attr_names()) == {'prodict_key', 'simple_key'}
r.prodict_key = Prodict(a=1)
print('r.prodict_key =', r.prodict_key)
print('r.prodict_key.a =', r.prodict_key.a)
print('type(r.prodict_key) =', type(r.prodict_key))
assert r.prodict_key == {'a': 1}
assert r.prodict_key.a == 1
assert type(r.prodict_key) == Prodict
def generate(cls, count=100, skip_ratio=0.0):
"""
A loop to iterate over to generate data.
:param count: Loop count
:param skip_ratio: Ratio of skipping a loop, useful to mimic realistic data
:return: Returns a Prodict (dot-dict)
"""
for index in range(count):
# if random.randint(0, 100) <= skip_ratio * 100:
# continue
yield Prodict()
def test_accept_generator(self):
"""
https://github.com/ramazanpolat/prodict/issues/18
"""
s = ';O2Sat:92;HR:62;RR:0'
# this works
dd1 = dict(x.split(':') for x in s.split(';') if ':' in x)
# this fails with TypeError: __init__() takes 1 positional argument but 2 were given
pd1 = Prodict(x.split(':') for x in s.split(';') if ':' in x)
print(pd1)
assert True
def __init__(
self,
in_channels : int,
out_channels : int,
stride : int = 1,
norm: str = "GN",
):
super(Bottleneck, self).__init__()
self.bottleneck = BasicBlock(in_channels,
out_channels,
stride=stride,
norm=norm,
activation=Prodict(NAME="ReLU", INPLACE=True))
self.init_parameters()
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
issues = []
updates = {}
with open(self.cfg_filename) as file:
applications: List[dict] = yaml.safe_load(file)
enabled_databases = [db_uid for db_uid, db in model.aws.databases.items()
if DbStatus[db.status] >= DbStatus.ENABLED]
for app in applications:
app_name = app['name']
db_ref = app['db']
db_id_list = wc_expand(db_ref, enabled_databases)
if not db_id_list:
issues.append(Issue(level=IssueLevel.ERROR, type='APP', id=app_name,
message=f"Not existing and enabled DB instance reference '{db_ref}'"))
continue
updates[app_name] = db_id_list
return Prodict(applications=updates), issues
def full_state(self):
started_at = self.started_at or 0
start_ts = 0
uptime_sec = 0
if started_at:
start_ts = started_at.timestamp
now_ts = arrow.now().timestamp
uptime_sec = now_ts - start_ts
uptime_sec = uptime_sec if uptime_sec > 0 else 0
return Prodict(running=self.running,
inband=self.inband_val,
started_at=start_ts * 1000,
created=self.create_ts,
uptime=uptime_sec * 1000,
state=self.status,
status=self.status)
def initial_model() -> Prodict:
config_dir = os.environ["SARI_CONFIG"]
regions = discover_regions(config_dir)
return Prodict(
system={
"config_dir": config_dir,
"proxy": os.environ.get("PROXY"),
},
aws={
"regions": regions,
"single_region": regions[0] if len(regions) == 1 else None,
"default_region": os.environ["AWS_REGION"],
"iam_roles": {
"trigger_run": os.environ["SARI_IAM_TRIGGER_ROLE_NAME"],
},
},
okta={
"organization": os.environ["OKTA_ORG_NAME"],
"api_token": os.environ["OKTA_API_TOKEN"],
},
bastion_host={
"hostname": os.environ["BH_HOSTNAME"],
"port": os.environ.get("BH_PORT"),
"admin_username": os.environ["BH_ADMIN_USERNAME"],
"admin_private_key": os.environ.get("BH_ADMIN_PRIVATE_KEY"),
"admin_key_filename": os.environ.get("BH_ADMIN_KEY_FILENAME"),
"admin_key_passphrase": os.environ["BH_ADMIN_KEY_PASSPHRASE"],
"proxy_username": os.environ["BH_PROXY_USERNAME"],
},
applications={},
job={
"next_transition": None,
},
grant_types={
"query": ["SELECT"],
"crud": ["SELECT", "UPDATE", "INSERT", "DELETE"],
},
master_password_defaults={
r"([a-z][a-z0-9-]+)": r"ssm:\1.master_password",
},
)
def test_mysql_gather_rds_status():
db_name = "db_blackwells"
username = "******"
# noinspection HardcodedPassword
password = "******"
# TODO: launch it at startup to use the same container to test rainy-day scenarios
# - connection refused
# - connection timeout
# - invalid username/password
# - invalid schema/database name
# - dropping dandling users
with MySqlContainer("mysql:5.7.17",
MYSQL_DATABASE=db_name,
MYSQL_USER=username,
MYSQL_PASSWORD=password) as mysql:
with ThreadPoolExecutor(max_workers=1) as executor:
mysql_gatherer = MySqlGatherer(executor, None)
model = Prodict(
aws={
"databases": {
"blackwells": {
"endpoint": {
"address": "localhost",
"port": mysql.get_exposed_port(3306),
},
"master_username": username,
"master_password": password,
}
}
})
updates, issues = mysql_gatherer.gather(model)
assert not issues
assert_dict_equals(
updates,
{"aws": {
"databases": {
"blackwells": {
"status": "ACCESSIBLE"
}
}
}})
def _gather_rds_status(self,
model: Prodict) -> Tuple[Prodict, List[Issue]]:
"""
For all RDS instances: check if it's possible to connect, authenticate with credentials, and get authorized
access to the primary DB. Reports each instance check on the console.
"""
databases = model.aws.databases
logger.info("Checking access to RDS instances:")
issues = []
futures = []
for db_uid, db in databases.items():
if 'endpoint' in db:
future = self.executor.submit(_check_mysql_instance, db)
else:
future = None
futures.append(future)
db_id_max_len = max(map(len, databases))
updates = {}
accessible = dict(status=DbStatus.ACCESSIBLE.name)
for db_uid, future in zip(databases, futures):
if future:
success, message = future.result(MYSQL_LOGIN_TIMEOUT)
color = ("red", "green")[success]
if success:
updates[db_uid] = accessible
else:
issues.append(
Issue(level=IssueLevel.ERROR,
type="DB",
id=db_uid,
message=message))
else:
success, message = (False, databases[db_uid].status)
color = "light-magenta"
leader = "." * (2 + db_id_max_len - len(db_uid))
logger.opt(colors=True).info(
f" {db_uid} {leader} <{color}>{message}</{color}>")
return Prodict(aws={"databases": updates}), issues
def test_deepcopy1():
root_node = Prodict(number=1, data="ROOT node", next=None)
copied = copy.deepcopy(root_node)
print("--root-node id:", id(root_node))
print(root_node)
print("--copied id:", id(copied))
print(copied)
print("--root_node.data")
print(type(root_node))
print(root_node.data)
print("--copied.data")
print(type(copied))
print(copied.data)
# have same dict
assert copied == root_node
# have different id
assert copied is not root_node
# have same type
assert type(root_node) is type(copied)
def short_info(self):
return Prodict(name=self.name,
short_id=self.short_id,
state=self.status,
status=self.status)
def labels(self):
if self.d.Labels:
return self.d.Labels
elif self.d.Config and self.d.Config.Labels:
return self.d.Config.Labels
return Prodict()
def def_labels(a_ports=[]):
return Prodict(inband='inband', ports=pack_ports(a_ports))
def test_okta_gather_user_info(self):
# Given:
model = initial_model()
model.okta.update(Prodict(users=USERS_CONFIG))
model.okta.users["*****@*****.**"] = {
"db_username": "******",
"permissions": {},
}
model.okta.users["*****@*****.**"] = {
"db_username": "******",
"permissions": {},
}
query_prefix = r"^limit=1&search=profile\.login\+eq\+"
@urlmatch(scheme="https",
netloc="acme.okta.com",
path=r"^/api/v1/users",
query=query_prefix)
def okta_user_info(url, request):
assert request.headers["Authorization"] == f"SSWS {OKTA_API_TOKEN}"
m = re.match(query_prefix + r'"(.*)@acme\.com"$',
unquote(url.query))
assert m
username = m.group(1)
user_file = Path(f"tests/data/users/{username}.json")
content = user_file.read_text() if user_file.exists() else "[]"
return response(status_code=200,
content=content,
headers={"Content-Type": "application/json"})
# When:
with ThreadPoolExecutor(max_workers=1) as executor:
okta_gatherer = OktaGatherer(OKTA_API_TOKEN, executor)
with HTTMock(okta_user_info):
resp, issues = okta_gatherer.gather(model)
# Then:
assert len(issues) == 3
assert issues[0].level == IssueLevel.ERROR
assert issues[0].type == "USER"
assert issues[0].id == "*****@*****.**"
assert issues[1].level == IssueLevel.ERROR
assert issues[1].type == "USER"
assert issues[1].id == "*****@*****.**"
assert issues[2].level == IssueLevel.ERROR
assert issues[2].type == "USER"
assert issues[2].id == "*****@*****.**"
assert_dict_equals(
resp, {
"okta": {
"users": {
"*****@*****.**": {
"status": "MISSING_SSH_PUBKEY",
},
"*****@*****.**": {
"status": "DEPROVISIONED",
},
"*****@*****.**": {
"status":
"ACTIVE",
"user_id":
"00m6q2lgisjgmFq64772",
"ssh_pubkey":
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfzjdkO1LKnS/it62jmw9tH4BznlnDCBrzaKguujJ15 "
"*****@*****.**",
},
"*****@*****.**": {
"status":
"ACTIVE",
"user_id":
"00u4subrvCRYYe2dx765",
"ssh_pubkey":
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGD13Dbe1QoYrFZqCue1TzGkzDSra9ZHzv8gZy9+vb0Y "
"*****@*****.**",
},
"*****@*****.**": {
"status": "ABSENT",
}
}
}
})
def crop(os=None, browser=None, device=None, **kwargs):
result = Prodict()
result.browser = browser
result.os = os
result.device = device
return result
from band import logger, settings, response, expose
from prodict import Prodict
from async_lru import alru_cache
from user_agents import parse
from asyncio import sleep
from pprint import pprint
import json
state = Prodict()
def crop(os=None, browser=None, device=None, **kwargs):
result = Prodict()
result.browser = browser
result.os = os
result.device = device
return result
@expose.enricher(keys=['in.gen.track'], props=dict(ua='td.ua'))
@alru_cache(maxsize=512)
async def enrich(ua, **params):
"""
Detect device type using User-Agent string
https://github.com/selwin/python-user-agents
"""
try:
parsed = parse(ua)
res = Prodict(
os_family=parsed.os.family,
os_version=list(v if isinstance(v, int)
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]:
issues = []
configured_databases = model.aws.databases
not_found = dict(status=DbStatus.ABSENT.name)
updates = {
db_uid: not_found
for db_uid in configured_databases
if db_uid.startswith(f"{self.aws.region}/")
}
for db in self.aws.rds_enum_databases(ENGINE_TYPE):
db_id = db["DBInstanceIdentifier"]
db_uid = f"{self.aws.region}/{db_id}"
if db_uid not in configured_databases:
try:
master_password, password_age = self.pwd_resolver.resolve(
db_id, None)
db_upd = {
"status": DbStatus.AUTO_ENABLED.name,
"permissions": {},
"master_password": master_password,
"password_age": password_age,
}
except Exception as e:
issues.append(
Issue(level=IssueLevel.WARNING,
type="DB",
id=db_uid,
message=f"Failed to auto-configure: {e}"))
continue
elif DbStatus[
configured_databases[db_uid].status] == DbStatus.ENABLED:
db_upd = {}
else:
del updates[db_uid]
continue
subnets_by_az = _get_subnets_by_az(db)
az = db.get(
"AvailabilityZone",
# Chose the AZ of the first subnet arbitrarily.
# Required for MOTO since AZ is not defined.
next(iter(subnets_by_az.keys())))
db_upd.update({
"db_name":
db["DBName"],
"master_username":
db["MasterUsername"],
"endpoint": {
"address": db["Endpoint"]["Address"],
"port": db["Endpoint"]["Port"],
},
"dbi_resource_id":
db["DbiResourceId"],
"availability_zone":
az,
"vpc_security_group_ids": [
sg["VpcSecurityGroupId"] for sg in db["VpcSecurityGroups"]
if sg["Status"] == "active"
],
"primary_subnet":
subnets_by_az[az][0]
})
updates[db_uid] = db_upd
for db_uid, db in updates.items():
if db == not_found:
issues.append(
Issue(level=IssueLevel.ERROR,
type="DB",
id=db_uid,
message="Not found in AWS"))
return Prodict(aws={"databases": updates}), issues