def test_dict_reserved_keys(): with tc.assertRaises(TypeError): Prodict(pop=5) pd2 = Prodict(pop1=5) with tc.assertRaises(TypeError): pd2.pop = 5
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: self._next_transition = model.job.next_transition issues = [] with _open(self.cfg_stream) as stream: users_list: List[dict] = yaml.safe_load(stream) or [] default_db_name = {db_uid: db.db_name for db_uid, db in model.aws.databases.items() if DbStatus[db.status] >= DbStatus.ENABLED} enabled_databases = list(default_db_name.keys()) users = {} databases = {} for user in users_list: login = user["login"] default_grant_type = user.get("default_grant_type", DEFAULT_GRANT_TYPE) try: permissions = self._parse_permissions( user.get("permissions", []), model.aws.single_region, default_grant_type, enabled_databases, default_db_name) users[login] = { "db_username": login[:MAX_DB_USERNAME_LENGTH], "permissions": permissions } for db_uid, grant_type in permissions.items(): databases.setdefault(db_uid, {"permissions": {}})["permissions"][login] = grant_type except ValueError as e: issues.append(Issue(level=IssueLevel.ERROR, type='USER', id=login, message=str(e))) updates = Prodict(okta={"users": users}, aws={"databases": databases}) if self._next_transition: updates.job = dict(next_transition=self._next_transition) return updates, issues
async def enrich(ua, **params): """ Detect device type using User-Agent string https://github.com/selwin/python-user-agents """ try: parsed = parse(ua) res = Prodict( os_family=parsed.os.family, os_version=list(v if isinstance(v, int) else 0 for v in parsed.os.version), browser_family=parsed.browser.family, browser_version=list(v if isinstance( v, int) else 0 for v in parsed.browser.version,), device_family=parsed.device.family, device_brand=parsed.device.brand, device_model=parsed.device.model) res.is_bot = int(parsed.is_bot) res.is_tables = int(parsed.is_tablet) res.is_mob = int(parsed.is_mobile) res.is_pc = int(parsed.is_pc) return res except Exception: logger.exception('handle ex') return response.error()
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: issues = [] updates = {} with open(self.cfg_filename) as file: services = yaml.safe_load(file) enabled_databases = [db_uid for db_uid, db in model.aws.databases.items() if DbStatus[db.status] >= DbStatus.ENABLED] for conn in services.get("glue_connections", []): db_ref = conn['db'] db_id_list = wc_expand(db_ref, enabled_databases) if not db_id_list: issues.append(Issue(level=IssueLevel.ERROR, type='GLUE', id=db_ref, message=f"Not existing and enabled DB instance reference '{db_ref}'")) continue pcr = conn.get("physical_connection_requirements", {}) supplied_db_names = conn.get("db_names") if isinstance(supplied_db_names, str): supplied_db_names = [supplied_db_names] grant_type = conn.get("grant_type", DEFAULT_GRANT_TYPE) for db_uid in db_id_list: db = model.aws.databases[db_uid] updates[db_uid] = { "db_names": supplied_db_names or [db.db_name], "grant_type": grant_type, "physical_connection_requirements": { "availability_zone": pcr.get("availability_zone", db.availability_zone), "security_group_id_list": pcr.get("security_group_id_list", db.vpc_security_group_ids), "subnet_id": pcr.get("subnet_id", db.primary_subnet), }, } return Prodict(aws={"glue_connections": updates}), issues
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: with open(self.cfg_filename) as file: rds_list: List[dict] = yaml.safe_load(file) issues = [] databases = {} for cfg_db in rds_list: db_id = cfg_db["id"] db_uid = f"{self.region}/{db_id}" enabled = _to_bool(cfg_db.setdefault("enabled", True)) if enabled: try: master_password, password_age = self.pwd_resolver.resolve(db_id, cfg_db.get("master_password")) db = { "status": DbStatus.ENABLED.name, "permissions": {}, "master_password": master_password, "password_age": password_age, } except Exception as e: issues.append(Issue(level=IssueLevel.ERROR, type="DB", id=db_uid, message=str(e))) continue else: db = dict(status=DbStatus.DISABLED.name) databases[db_uid] = db return Prodict(aws={"databases": databases}), issues
def __init__(self, cfg: List[Union[str, Path, dict]]): self._instance = Prodict() for item in cfg: self.update(item) self._ensure_has_all_params(Config.REQUIRED_PARAMS) logger.debug(f"Configuration loaded from {cfg}")
def __init__(self, in_channels: int, out_channels: int, kernel_size: int = 1, padding: int = 0, stride: int = 1, num_groups: int = 1, norm: str = "GN", gate_activation: str = "ReTanH", gate_activation_kargs: dict = None): super(DynamicBottleneck, self).__init__() self.num_groups = num_groups self.norm = norm self.in_channels = in_channels self.out_channels = out_channels self.bottleneck = BasicBlock(in_channels, out_channels, stride=stride, norm=norm, activation=Prodict(NAME="ReLU", INPLACE=True)) self.gate = SpatialGate(in_channels, num_groups=num_groups, kernel_size=kernel_size, padding=padding, stride=stride, gate_activation=gate_activation, gate_activation_kargs=gate_activation_kargs, get_running_cost=self.get_running_cost) self.init_parameters()
async def chech(p, params): asyncio.sleep(random()*1) check = Prodict(success=0, region=p.cityRu) check.update({k: p[k] for k in copyattrs}) try: state.myiter() async with aiohttp.ClientSession() as chs: proxy_auth = aiohttp.BasicAuth(p.user, p.password) hostport = f"http://{p.host.strip()}:{p.port}" async with chs.get(test_url, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as pr: check.responseCode = pr.status if pr.status == 200: check.success = 1 check.contentSize = len(await pr.text()) async with chs.get(check_ip, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as ip_r: if ip_r.status == 200: check.extIp = (await ip_r.text()).strip() state.succ() except ClientConnectorError: logger.warn('connection error: %s:%s', p.host, p.port) except asyncio.TimeoutError: logger.warn('asyncio timeout') except TimeoutError: logger.warn('timeout') except Exception: logger.exception('check err') await asyncio.sleep(1) try: async with aiohttp.ClientSession() as ss: async with ss.post(params.notify, json=check, timeout=10) as wh_r: if wh_r.status != 200: logger.error('webhook failed') except Exception: logger.exception('sending webhook err')
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: """ Check if the users exist and retrieve their corresponding user_id and ssh_pubkey. """ okta = model.okta session = async_retryable_session(self.executor) futures = [] searcher = jmespath.compile( "[*].[id, status, profile.sshPubKey] | [0]") for login in okta.users: future = session.get( f"https://{okta.organization}.okta.com/api/v1/users?limit=1&search=profile.login+eq+" + urllib.parse.quote(f'"{login}"'), headers=(self._http_headers())) futures.append(future) issues = [] users_ext = {} logger.info(f"Checking Okta {okta.organization.capitalize()}'s Users:") login_max_len = max(map(len, okta.users), default=0) for login, future in zip(okta.users, futures): result = future.result() result.raise_for_status() json_response = json.loads(result.content.decode()) match = searcher.search(json_response) user_data = {} if match: user_id, status, ssh_pubkey = match if status != "ACTIVE": err_msg = f"status={status}" elif ssh_pubkey: err_msg = None user_data = { "user_id": user_id, "ssh_pubkey": ssh_pubkey, } else: status = "MISSING_SSH_PUBKEY" err_msg = "Missing SSH PubKey" else: status = "ABSENT" err_msg = "Not found in OKTA" user_data["status"] = status if err_msg: color = "red" issues.append( Issue(level=IssueLevel.ERROR, type="USER", id=login, message=err_msg)) else: color = "green" leader = "." * (2 + login_max_len - len(login)) logger.opt(colors=True).info( f" {login} {leader} <{color}>{status}</{color}>") users_ext[login] = user_data return Prodict(okta={"users": users_ext}), issues
def __init__(self, cfg: Iterable[Union[str, Path, dict]] = None): cfg = cfg or [] self._instance = Prodict() for item in cfg: if item: self.update(item) self._ensure_has_all_params(Config.REQUIRED_PARAMS) logger.debug(f"Configuration loaded from {cfg}")
def test_issue15(self): """url: https://github.com/ramazanpolat/prodict/issues/15 if the payload has a attribute named 'self' then we get a TypeError: TypeError: __init__() got multiple values for argument 'self' """ try: p = Prodict(self=1) assert True except TypeError: assert False
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: """Loads the optional `custom.yaml` from the configuration directory.""" issues = [] custom_yaml = f"{model.system.config_dir}/custom.yaml" if os.path.exists(custom_yaml): with open(custom_yaml) as file: custom = yaml.safe_load(file) # TODO: validate else: custom = {} return Prodict(custom=custom), issues
def test_pickle(self): try: encoded = pickle.dumps(Prodict(a=42)) decoded = pickle.loads(encoded) assert decoded.a == 42 # p = Prodict(a=1, b=2) # encoded = pickle.dumps(p) # print(encoded) # decoded = pickle.loads(encoded) # print(decoded) except: assert False
def test_recursive_annotations1(): r = Recursive() assert r == {} assert set(r.attr_names()) == {'prodict_key', 'simple_key'} r.prodict_key = Prodict(a=1) print('r.prodict_key =', r.prodict_key) print('r.prodict_key.a =', r.prodict_key.a) print('type(r.prodict_key) =', type(r.prodict_key)) assert r.prodict_key == {'a': 1} assert r.prodict_key.a == 1 assert type(r.prodict_key) == Prodict
def generate(cls, count=100, skip_ratio=0.0): """ A loop to iterate over to generate data. :param count: Loop count :param skip_ratio: Ratio of skipping a loop, useful to mimic realistic data :return: Returns a Prodict (dot-dict) """ for index in range(count): # if random.randint(0, 100) <= skip_ratio * 100: # continue yield Prodict()
def test_accept_generator(self): """ https://github.com/ramazanpolat/prodict/issues/18 """ s = ';O2Sat:92;HR:62;RR:0' # this works dd1 = dict(x.split(':') for x in s.split(';') if ':' in x) # this fails with TypeError: __init__() takes 1 positional argument but 2 were given pd1 = Prodict(x.split(':') for x in s.split(';') if ':' in x) print(pd1) assert True
def __init__( self, in_channels : int, out_channels : int, stride : int = 1, norm: str = "GN", ): super(Bottleneck, self).__init__() self.bottleneck = BasicBlock(in_channels, out_channels, stride=stride, norm=norm, activation=Prodict(NAME="ReLU", INPLACE=True)) self.init_parameters()
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: issues = [] updates = {} with open(self.cfg_filename) as file: applications: List[dict] = yaml.safe_load(file) enabled_databases = [db_uid for db_uid, db in model.aws.databases.items() if DbStatus[db.status] >= DbStatus.ENABLED] for app in applications: app_name = app['name'] db_ref = app['db'] db_id_list = wc_expand(db_ref, enabled_databases) if not db_id_list: issues.append(Issue(level=IssueLevel.ERROR, type='APP', id=app_name, message=f"Not existing and enabled DB instance reference '{db_ref}'")) continue updates[app_name] = db_id_list return Prodict(applications=updates), issues
def full_state(self): started_at = self.started_at or 0 start_ts = 0 uptime_sec = 0 if started_at: start_ts = started_at.timestamp now_ts = arrow.now().timestamp uptime_sec = now_ts - start_ts uptime_sec = uptime_sec if uptime_sec > 0 else 0 return Prodict(running=self.running, inband=self.inband_val, started_at=start_ts * 1000, created=self.create_ts, uptime=uptime_sec * 1000, state=self.status, status=self.status)
def initial_model() -> Prodict: config_dir = os.environ["SARI_CONFIG"] regions = discover_regions(config_dir) return Prodict( system={ "config_dir": config_dir, "proxy": os.environ.get("PROXY"), }, aws={ "regions": regions, "single_region": regions[0] if len(regions) == 1 else None, "default_region": os.environ["AWS_REGION"], "iam_roles": { "trigger_run": os.environ["SARI_IAM_TRIGGER_ROLE_NAME"], }, }, okta={ "organization": os.environ["OKTA_ORG_NAME"], "api_token": os.environ["OKTA_API_TOKEN"], }, bastion_host={ "hostname": os.environ["BH_HOSTNAME"], "port": os.environ.get("BH_PORT"), "admin_username": os.environ["BH_ADMIN_USERNAME"], "admin_private_key": os.environ.get("BH_ADMIN_PRIVATE_KEY"), "admin_key_filename": os.environ.get("BH_ADMIN_KEY_FILENAME"), "admin_key_passphrase": os.environ["BH_ADMIN_KEY_PASSPHRASE"], "proxy_username": os.environ["BH_PROXY_USERNAME"], }, applications={}, job={ "next_transition": None, }, grant_types={ "query": ["SELECT"], "crud": ["SELECT", "UPDATE", "INSERT", "DELETE"], }, master_password_defaults={ r"([a-z][a-z0-9-]+)": r"ssm:\1.master_password", }, )
def test_mysql_gather_rds_status(): db_name = "db_blackwells" username = "******" # noinspection HardcodedPassword password = "******" # TODO: launch it at startup to use the same container to test rainy-day scenarios # - connection refused # - connection timeout # - invalid username/password # - invalid schema/database name # - dropping dandling users with MySqlContainer("mysql:5.7.17", MYSQL_DATABASE=db_name, MYSQL_USER=username, MYSQL_PASSWORD=password) as mysql: with ThreadPoolExecutor(max_workers=1) as executor: mysql_gatherer = MySqlGatherer(executor, None) model = Prodict( aws={ "databases": { "blackwells": { "endpoint": { "address": "localhost", "port": mysql.get_exposed_port(3306), }, "master_username": username, "master_password": password, } } }) updates, issues = mysql_gatherer.gather(model) assert not issues assert_dict_equals( updates, {"aws": { "databases": { "blackwells": { "status": "ACCESSIBLE" } } }})
def _gather_rds_status(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: """ For all RDS instances: check if it's possible to connect, authenticate with credentials, and get authorized access to the primary DB. Reports each instance check on the console. """ databases = model.aws.databases logger.info("Checking access to RDS instances:") issues = [] futures = [] for db_uid, db in databases.items(): if 'endpoint' in db: future = self.executor.submit(_check_mysql_instance, db) else: future = None futures.append(future) db_id_max_len = max(map(len, databases)) updates = {} accessible = dict(status=DbStatus.ACCESSIBLE.name) for db_uid, future in zip(databases, futures): if future: success, message = future.result(MYSQL_LOGIN_TIMEOUT) color = ("red", "green")[success] if success: updates[db_uid] = accessible else: issues.append( Issue(level=IssueLevel.ERROR, type="DB", id=db_uid, message=message)) else: success, message = (False, databases[db_uid].status) color = "light-magenta" leader = "." * (2 + db_id_max_len - len(db_uid)) logger.opt(colors=True).info( f" {db_uid} {leader} <{color}>{message}</{color}>") return Prodict(aws={"databases": updates}), issues
def test_deepcopy1(): root_node = Prodict(number=1, data="ROOT node", next=None) copied = copy.deepcopy(root_node) print("--root-node id:", id(root_node)) print(root_node) print("--copied id:", id(copied)) print(copied) print("--root_node.data") print(type(root_node)) print(root_node.data) print("--copied.data") print(type(copied)) print(copied.data) # have same dict assert copied == root_node # have different id assert copied is not root_node # have same type assert type(root_node) is type(copied)
def short_info(self): return Prodict(name=self.name, short_id=self.short_id, state=self.status, status=self.status)
def labels(self): if self.d.Labels: return self.d.Labels elif self.d.Config and self.d.Config.Labels: return self.d.Config.Labels return Prodict()
def def_labels(a_ports=[]): return Prodict(inband='inband', ports=pack_ports(a_ports))
def test_okta_gather_user_info(self): # Given: model = initial_model() model.okta.update(Prodict(users=USERS_CONFIG)) model.okta.users["*****@*****.**"] = { "db_username": "******", "permissions": {}, } model.okta.users["*****@*****.**"] = { "db_username": "******", "permissions": {}, } query_prefix = r"^limit=1&search=profile\.login\+eq\+" @urlmatch(scheme="https", netloc="acme.okta.com", path=r"^/api/v1/users", query=query_prefix) def okta_user_info(url, request): assert request.headers["Authorization"] == f"SSWS {OKTA_API_TOKEN}" m = re.match(query_prefix + r'"(.*)@acme\.com"$', unquote(url.query)) assert m username = m.group(1) user_file = Path(f"tests/data/users/{username}.json") content = user_file.read_text() if user_file.exists() else "[]" return response(status_code=200, content=content, headers={"Content-Type": "application/json"}) # When: with ThreadPoolExecutor(max_workers=1) as executor: okta_gatherer = OktaGatherer(OKTA_API_TOKEN, executor) with HTTMock(okta_user_info): resp, issues = okta_gatherer.gather(model) # Then: assert len(issues) == 3 assert issues[0].level == IssueLevel.ERROR assert issues[0].type == "USER" assert issues[0].id == "*****@*****.**" assert issues[1].level == IssueLevel.ERROR assert issues[1].type == "USER" assert issues[1].id == "*****@*****.**" assert issues[2].level == IssueLevel.ERROR assert issues[2].type == "USER" assert issues[2].id == "*****@*****.**" assert_dict_equals( resp, { "okta": { "users": { "*****@*****.**": { "status": "MISSING_SSH_PUBKEY", }, "*****@*****.**": { "status": "DEPROVISIONED", }, "*****@*****.**": { "status": "ACTIVE", "user_id": "00m6q2lgisjgmFq64772", "ssh_pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfzjdkO1LKnS/it62jmw9tH4BznlnDCBrzaKguujJ15 " "*****@*****.**", }, "*****@*****.**": { "status": "ACTIVE", "user_id": "00u4subrvCRYYe2dx765", "ssh_pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGD13Dbe1QoYrFZqCue1TzGkzDSra9ZHzv8gZy9+vb0Y " "*****@*****.**", }, "*****@*****.**": { "status": "ABSENT", } } } })
def crop(os=None, browser=None, device=None, **kwargs): result = Prodict() result.browser = browser result.os = os result.device = device return result
from band import logger, settings, response, expose from prodict import Prodict from async_lru import alru_cache from user_agents import parse from asyncio import sleep from pprint import pprint import json state = Prodict() def crop(os=None, browser=None, device=None, **kwargs): result = Prodict() result.browser = browser result.os = os result.device = device return result @expose.enricher(keys=['in.gen.track'], props=dict(ua='td.ua')) @alru_cache(maxsize=512) async def enrich(ua, **params): """ Detect device type using User-Agent string https://github.com/selwin/python-user-agents """ try: parsed = parse(ua) res = Prodict( os_family=parsed.os.family, os_version=list(v if isinstance(v, int)
def gather(self, model: Prodict) -> Tuple[Prodict, List[Issue]]: issues = [] configured_databases = model.aws.databases not_found = dict(status=DbStatus.ABSENT.name) updates = { db_uid: not_found for db_uid in configured_databases if db_uid.startswith(f"{self.aws.region}/") } for db in self.aws.rds_enum_databases(ENGINE_TYPE): db_id = db["DBInstanceIdentifier"] db_uid = f"{self.aws.region}/{db_id}" if db_uid not in configured_databases: try: master_password, password_age = self.pwd_resolver.resolve( db_id, None) db_upd = { "status": DbStatus.AUTO_ENABLED.name, "permissions": {}, "master_password": master_password, "password_age": password_age, } except Exception as e: issues.append( Issue(level=IssueLevel.WARNING, type="DB", id=db_uid, message=f"Failed to auto-configure: {e}")) continue elif DbStatus[ configured_databases[db_uid].status] == DbStatus.ENABLED: db_upd = {} else: del updates[db_uid] continue subnets_by_az = _get_subnets_by_az(db) az = db.get( "AvailabilityZone", # Chose the AZ of the first subnet arbitrarily. # Required for MOTO since AZ is not defined. next(iter(subnets_by_az.keys()))) db_upd.update({ "db_name": db["DBName"], "master_username": db["MasterUsername"], "endpoint": { "address": db["Endpoint"]["Address"], "port": db["Endpoint"]["Port"], }, "dbi_resource_id": db["DbiResourceId"], "availability_zone": az, "vpc_security_group_ids": [ sg["VpcSecurityGroupId"] for sg in db["VpcSecurityGroups"] if sg["Status"] == "active" ], "primary_subnet": subnets_by_az[az][0] }) updates[db_uid] = db_upd for db_uid, db in updates.items(): if db == not_found: issues.append( Issue(level=IssueLevel.ERROR, type="DB", id=db_uid, message="Not found in AWS")) return Prodict(aws={"databases": updates}), issues