def listResources(label, targetFolder, url, method, payload, current, folderAnnotation, resource, logger): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current) if namespace == "ALL": ret = getattr(v1, _list_for_all_namespaces[resource])() else: ret = getattr(v1, _list_namespaced[resource])(namespace=namespace) for sec in ret.items: destFolder = targetFolder metadata = sec.metadata if metadata.labels is None: continue logger.info("Working on %s: %s/%s" % (resource, metadata.namespace, metadata.name)) if label in sec.metadata.labels.keys(): logger.info("Found %s with label" % resource) if sec.metadata.annotations is not None: if folderAnnotation in sec.metadata.annotations.keys(): destFolder = sec.metadata.annotations[folderAnnotation] dataMap = sec.data if dataMap is None: logger.info("No data field in %s" % resource) continue if label in sec.metadata.labels.keys(): for data_key in dataMap.keys(): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource, logger) writeTextToFile(destFolder, filename, filedata) if url is not None: request(url, method, logger, payload)
def generate_advert_county_urls(): """ Generator function navigates through the pages of real estate adverts and extracts all the links to the adverts. """ # Build the BS object from the page source page_source = None while not page_source: page_source = helpers.request(_BASE_URL, proxies=False) if page_source.status_code != 200: page_source = None page_source = page_source.text.encode('utf-8') soup = BeautifulSoup(page_source, 'lxml') # Extract the links to the offers filtered by provider for provider_url in soup.findAll('a', {'class': 'fleft tab tdnone topTabOffer'}): provider_url = provider_url['href'] provider_page_source = helpers.request(provider_url, proxies=False) if not provider_page_source: continue provider_page_source = provider_page_source.text.encode('utf-8') provider_soup = BeautifulSoup(provider_page_source, 'lxml') for county_url in provider_soup.select('div.locationlinks.margintop10 li a'): if county_url.has_attr('href'): yield county_url['href']
def test_error_update_unauthorized(client): not_found_response = helpers.request( client, 'PATCH', '/widgets/1?user_id=bar', { 'id': '1', 'owner_id': 'bar', 'name': "Updated", }, ) assert not_found_response.status_code == 404 forbidden_response = helpers.request( client, 'PATCH', '/widgets/3?user_id=bar', { 'id': '3', 'owner_id': 'bar', 'name': "Updated", }, ) assert forbidden_response.status_code == 403 assert helpers.get_errors(forbidden_response) == [{ 'code': 'invalid_user' }]
def listResources(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", currentNamespace) # Filter resources based on label and value or just label labelSelector=f"{label}={labelValue}" if labelValue else label if namespace == "ALL": ret = getattr(v1, _list_for_all_namespaces[resource])(label_selector=labelSelector) else: ret = getattr(v1, _list_namespaced[resource])(namespace=namespace, label_selector=labelSelector) # For all the found resources for sec in ret.items: metadata = sec.metadata print(f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}") # Get the destination folder destFolder = _get_destination_folder(metadata, targetFolder, folderAnnotation) # Check if it's an empty ConfigMap or Secret dataMap = sec.data if dataMap is None: print(f"{timestamp()} No data field in {resource}") continue # Each key on the data is a file for data_key in dataMap.keys(): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource) writeTextToFile(destFolder, filename, filedata) if url: request(url, method, payload)
def remove_stats(self, *args): url = f"https://api.trello.com/1/cards/{self.id}" for stat in args: try: del self.stats[stat] except KeyError: continue self.desc = hlp.format_desc(self.stats) hlp.request("PUT", url, desc=self.desc)
def add_service(config, realm): name = config['name'] # service name host = config['host'] # service host print(f'\nAdding realm "{realm}" to service "{name}"') # OIDC plugin settings (same for all OIDC endpoints) oidc_data = _get_service_oidc_payload(name, realm) ep_types = [EPT_PUBLIC, EPT_OIDC] for ep_type in ep_types: print(f' Adding "{ep_type}" endpoints') endpoints = config.get(f'{ep_type}_endpoints', []) for ep in endpoints: ep_name = ep['name'] ep_url = ep['url'] service_name = f'{name}_{ep_type}_{ep_name}' data = { 'name': service_name, 'url': f'{host}{ep_url}', } try: request(method='post', url=f'{KONG_URL}/services/', data=data) print(f' + Added service "{ep_name}"') except HTTPError: print(f' - Could not add service "{ep_name}"') ROUTE_URL = f'{KONG_URL}/services/{service_name}/routes' path = ep.get('route_path') or f'/{realm}/{name}{ep_url}' route_data = { 'paths': [ path, ], 'strip_path': ep.get('strip_path', 'false'), } try: route_info = request(method='post', url=ROUTE_URL, data=route_data) # OIDC routes are protected using the "kong-oidc-auth" plugin if ep_type == EPT_OIDC: protected_route_id = route_info['id'] request( method='post', url=f'{KONG_URL}/routes/{protected_route_id}/plugins', data=oidc_data, ) print(f' + Added route {ep_url} >> {path}') except HTTPError: print(f' - Could not add route {ep_url} >> {path}') print(f'Service "{name}" now being served by kong for realm "{realm}"\n')
def archive_cards(self): url = f"https://api.trello.com/1/lists/{self.id}/archiveAllCards" hlp.request("POST", url) card_names = [card.name for card in self.cards] print(f"All cards archived in list {self.name}: {card_names}") self._log_date() self.board._update_task_file() self.cards = [] self.board._get_cards()
def _watch_resource_iterator(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource, uniqueFilenames): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", currentNamespace) # Filter resources based on label and value or just label labelSelector=f"{label}={labelValue}" if labelValue else label if namespace == "ALL": stream = watch.Watch().stream(getattr(v1, _list_for_all_namespaces[resource]), label_selector=labelSelector) else: stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace, label_selector=labelSelector) # Process events for event in stream: metadata = event["object"].metadata print(f"{timestamp()} Working on {resource} {metadata.namespace}/{metadata.name}") # Get the destination folder destFolder = _get_destination_folder(metadata, targetFolder, folderAnnotation) # Check if it's an empty ConfigMap or Secret dataMap = event["object"].data if dataMap is None: print(f"{timestamp()} {resource} does not have data.") continue eventType = event["type"] # Each key on the data is a file for data_key in dataMap.keys(): print(f"{timestamp()} File in {resource} {data_key} {eventType}") if (eventType == "ADDED") or (eventType == "MODIFIED"): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource) if uniqueFilenames: filename = uniqueFilename(filename = filename, namespace = metadata.namespace, resource = resource, resource_name = metadata.name) writeTextToFile(destFolder, filename, filedata) else: # Get filename from event filename = data_key[:-4] if data_key.endswith(".url") else data_key if uniqueFilenames: filename = uniqueFilename(filename = filename, namespace = metadata.namespace, resource = resource, resource_name = metadata.name) removeFile(destFolder, filename) if url: request(url, method, payload)
def sort_list(self): prefer_order = lambda x: ( x.stats.get("priority", 999), x.labels[0]["name"], x.due, x.name) self.cards = sorted(self.cards, key=prefer_order) for rank, card in enumerate(self.cards, start=1): url = f"https://api.trello.com/1/cards/{card.id}" hlp.request("PUT", url, pos=rank)
def list_resources(label, label_value, target_folder, request_url, request_method, request_payload, namespace, folder_annotation, resource, unique_filenames, script, enable_5xx, ignore_already_processed): v1 = client.CoreV1Api() # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label additional_args = {'label_selector': label_selector} if namespace != "ALL": additional_args['namespace'] = namespace ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args) files_changed = False # For all the found resources for item in ret.items: metadata = item.metadata # Ignore already processed resource # Avoid numerous logs about useless resource processing each time the LIST loop reconnects if ignore_already_processed: if _resources_version_map.get( metadata.namespace + metadata.name) == metadata.resource_version: # print(f"{timestamp()} Ignoring {resource} {metadata.namespace}/{metadata.name}") continue _resources_version_map[metadata.namespace + metadata.name] = metadata.resource_version print( f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}" ) # Get the destination folder dest_folder = _get_destination_folder(metadata, target_folder, folder_annotation) if resource == RESOURCE_CONFIGMAP: files_changed |= _process_config_map(dest_folder, item, resource, unique_filenames, enable_5xx) else: files_changed = _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx) if script and files_changed: execute(script) if request_url and files_changed: request(request_url, request_method, enable_5xx, request_payload)
def _watch_resource_iterator(label, label_value, target_folder, url, method, payload, current_namespace, folder_annotation, resource, unique_filenames, script): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current_namespace) # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label if namespace == "ALL": stream = watch.Watch().stream(getattr( v1, _list_for_all_namespaces[resource]), label_selector=label_selector, _request_timeout=60) else: stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace, label_selector=label_selector, _request_timeout=60) # Process events for event in stream: item = event["object"] metadata = item.metadata event_type = event["type"] print( f"{timestamp()} Working on {event_type} {resource} {metadata.namespace}/{metadata.name}" ) files_changed = False # Get the destination folder dest_folder = _get_destination_folder(metadata, target_folder, folder_annotation) item_removed = event_type == "DELETED" if resource == RESOURCE_CONFIGMAP: files_changed |= _process_config_map(dest_folder, item, resource, unique_filenames, item_removed) else: files_changed |= _process_secret(dest_folder, item, resource, unique_filenames, item_removed) if script and files_changed: execute(script) if url and files_changed: request(url, method, payload)
def create_card(self): self.assign_due_date() self.create_card_body() inbox = self._board.lists["Inbox"] url = "https://api.trello.com/1/cards" params = { "idList": inbox.id, "name": self.name, "idLabels": self._label_ids, "due": self.due, "desc": hlp.format_desc(self.card_body) } hlp.request("POST", url, **params) print(f"Posted card: {self.name} (due {self.due.date()})")
def test_resource_rules(app, views, client): api = Api(app) api.add_resource( base_rule='/widget/<id>', base_view=views['widget'], alternate_rule='/widgets', alternate_view=views['widget_list'], ) get_response = client.get('/widget/1') assert get_response.status_code == 200 assert helpers.get_data(get_response) == { 'id': '1', } post_response = helpers.request( client, 'POST', '/widgets', {}, ) assert post_response.status_code == 201 assert post_response.headers['Location'] == 'http://localhost/widget/2' assert helpers.get_data(post_response) == { 'id': '2', }
def _get_labels(self): url = f"https://api.trello.com/1/boards/{self.board_id}/labels/" labels = hlp.request("GET", url) label_map = {label["name"]: label["id"] for label in labels} self.labels = label_map self.label_names = {id: name for name, id in self.labels.items()} print("Fetched labels")
def test_many(client): response = request( client, 'PUT', '/parents/1', { 'id': '1', 'name': "Updated Parent", 'children': [ {'id': '1'}, {'id': '2'}, ], }, ) assert_response(response, 200, { 'id': '1', 'name': "Updated Parent", 'children': [ { 'id': '1', 'name': "Child 1", }, { 'id': '2', 'name': "Child 2", }, ], })
def test_many(client): response = helpers.request( client, 'PUT', '/parents/1', { 'id': '1', 'name': "Updated Parent", 'children': [ {'id': '1'}, {'id': '2'}, ], }, ) assert helpers.get_data(response) == { 'id': '1', 'name': "Updated Parent", 'children': [ { 'id': '1', 'name': "Child 1", }, { 'id': '2', 'name': "Child 2", }, ], }
def test_many_nested(client): response = request( client, 'PUT', '/nested_parents/1', { 'id': '1', 'name': "Updated Parent", 'children': [ {'name': "Child 3"}, {'name': "Child 4"}, ], }, ) assert_response(response, 200, { 'id': '1', 'name': "Updated Parent", 'children': [ { 'id': '3', 'name': "Child 3", }, { 'id': '4', 'name': "Child 4", }, ], })
def generate_advert_urls(): """ Generate all the adverd offer urls from all the indexed pages within the real estate 'super category' of the webside. """ for category_url in generate_advert_category_urls(): url_template = category_url + '&page={}' for index in itertools.count(1): # Access the page and get the source page_url = url_template.format(index) page_source = helpers.request(page_url, proxies=False) page_source = page_source.text.encode('utf-8') page_soup = BeautifulSoup(page_source, 'lxml') try: # Extract urls from the page for offer_url in page_soup.select('td.offer a.marginright5.link'): if offer_url.has_attr('href') and offer_url['href'] not in _EXTRACTED_OFFER_URLS: _EXTRACTED_OFFER_URLS.append(offer_url['href']) yield offer_url['href'] # Look exit condition last_page = page_soup.select('a.link.pageNextPrev') if (len(last_page) < 2 and index > 1) or not last_page: break except Exception as e: print(e.with_traceback(e.__traceback__)) with open('urls_error.htm', 'wt') as urls_error: print('an error has occurred') urls_error.write(page_soup.prettify()) break
def call(self): # Build the URL for the API (they are RESTful) kvFormat = '{}={}' params = '' # Now turn JSON like ``"a":"b"`` into URL parameters like ``a=b`` if self._payload is not {}: params = '&'.join([ kvFormat.format(key, str(self._payload[key])) for key in sorted(self._payload.keys()) ]) url = self.__host % (self._path, params) else: url = self.__host_no_params # Obtain the request signature timestamp = str(int((helpers.time.time() + helpers.timeOffset) * 1000)) signature = signer.sign(self._path, timestamp, params) # Generate the headers, fill with the request signature headers = { 'KC-API-KEY': signer.key, 'KC-API-NONCE': timestamp, 'KC-API-SIGNATURE': signature, 'Content-Type': 'application/x-www-form-urlencoded' } # Pass the actual request off to helpers, since they do error checking and more return helpers.request(self._method, url, headers, self)
def is_alive(self): logging.main_logger.debug("[sim] 'is_alive' called") self.alive = False # check if there is a pid file check_pid = False if self.pid_file != '': check_pid = os.path.isfile(self.pid_file) # check if the sim is responding to simstatus check_simstatus = False from helpers import request req = request('http://127.0.0.1:' + str(self.port) + '/simstatus') if req: # check if status is 200 if req.getcode() == 200: if req.read() == 'OK': check_simstatus = True if check_pid and check_simstatus: self.alive = True logging.main_logger.debug("[sim] sim %s is alive" % (self.path)) return True else: logging.main_logger.debug("[sim] sim %s is not alive" % (self.path)) return False
def _watch_resource_iterator(label, targetFolder, url, method, payload, current, folderAnnotation, resource): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current) if namespace == "ALL": stream = watch.Watch().stream( getattr(v1, _list_for_all_namespaces[resource])) else: stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace) for event in stream: destFolder = targetFolder metadata = event['object'].metadata if metadata.labels is None: continue print(f'Working on {resource} {metadata.namespace}/{metadata.name}') if label in event['object'].metadata.labels.keys(): print(f"{resource} with label found") if event['object'].metadata.annotations is not None: if folderAnnotation in event[ 'object'].metadata.annotations.keys(): destFolder = event['object'].metadata.annotations[ folderAnnotation] print('Found a folder override annotation, ' f'placing the {resource} in: {destFolder}') dataMap = event['object'].data if dataMap is None: print(f"{resource} does not have data.") continue eventType = event['type'] for data_key in dataMap.keys(): print(f"File in {resource} {data_key} {eventType}") if (eventType == "ADDED") or (eventType == "MODIFIED"): filename, filedata = _get_file_data_and_name( data_key, dataMap[data_key], resource) writeTextToFile(destFolder, filename, filedata) if url is not None: request(url, method, payload) else: filename = data_key[:-4] if data_key.endswith( ".url") else data_key removeFile(destFolder, filename) if url is not None: request(url, method, payload)
def _watch_resource_iterator(label, label_value, target_folder, request_url, request_method, request_payload, namespace, folder_annotation, resource, unique_filenames, script, enable_5xx): v1 = client.CoreV1Api() # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label additional_args = {'label_selector': label_selector} if namespace != "ALL": additional_args['namespace'] = namespace stream = watch.Watch().stream( getattr(v1, _list_namespace[namespace][resource]), **additional_args) # Process events for event in stream: item = event["object"] metadata = item.metadata event_type = event["type"] print( f"{timestamp()} Working on {event_type} {resource} {metadata.namespace}/{metadata.name}" ) files_changed = False # Get the destination folder dest_folder = _get_destination_folder(metadata, target_folder, folder_annotation) item_removed = event_type == "DELETED" if resource == RESOURCE_CONFIGMAP: files_changed |= _process_config_map(dest_folder, item, resource, unique_filenames, enable_5xx, item_removed) else: files_changed |= _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx, item_removed) if script and files_changed: execute(script) if request_url and files_changed: request(request_url, request_method, enable_5xx, request_payload)
def test_update(client): update_response = helpers.request(client, "PATCH", "/widgets/1/2", {"id_1": "1", "id_2": "2", "name": "Qux"}) assert update_response.status_code == 204 retrieve_response = client.get("/widgets/1/2") assert retrieve_response.status_code == 200 assert helpers.get_data(retrieve_response) == {"id_1": "1", "id_2": "2", "name": "Qux"}
def move_card(self, new_list_): url = f"https://api.trello.com/1/cards/{self.id}" new_list = self.list.board.lists[new_list_] r = hlp.request("PUT", url, idList=new_list.id) if r.status_code == 200: self.list.cards.remove(self) self.list = new_list self.list.cards.append(self) print(f"Moved card {self.name} to {self.list.name} (due {self.due})")
def test_create(client): response = helpers.request( client, 'POST', '/widgets?user_id=foo', { 'owner_id': 'foo', 'name': "Created", }, ) assert response.status_code == 201
def test_update_update_missing(client): response = request( client, 'PUT', '/widgets_create_missing/4?user_id=foo&owner_id=foo', { 'id': '4', 'name': "Created", } ) assert_response(response, 204)
def test_limit_offset_create(client): response = request( client, 'POST', '/limit_offset_widgets', { 'size': 1, }, ) assert 'meta' not in get_body(response)
def get_list_cards(self): self.cards = [] url = f'https://api.trello.com/1/lists/{self.id}/cards' fields = ["id","name","desc","due","labels"] card_list = hlp.request("GET", url, fields=fields) for card_input in card_list: card_input["due"] = hlp.localize_ts(card_input["due"]) card_input["list"] = self self.cards.append(Card(card_input)) self.time_sum()
def test_create(client): response = request( client, 'POST', '/widgets?user_id=foo', { 'owner_id': 'foo', 'name': "Created", }, ) assert_response(response, 201)
def test_page_create(client): response = request( client, 'POST', '/page_widgets', { 'size': 1, }, ) assert 'meta' not in get_body(response)
def test_update_tenant_id(client, credentials, result): response = request( client, 'PATCH', '/widgets/2', { 'id': '2', 'tenant_id': TENANT_ID_1, }, query_string=credentials, ) assert_response(response, result)
def get_checklists(self): self.checklists = defaultdict(list) url = f"https://api.trello.com/1/cards/{self.id}/checklists" raw = hlp.request("GET", url) for checklist in raw: name = checklist["name"] check_items = sorted(checklist["checkItems"], key=lambda x: x["pos"]) for checkitem in check_items: item_name = checkitem["name"] self.checklists[name].append(item_name)
def _get_lists(self): url = f"https://api.trello.com/1/boards/{self.board_id}/lists/" lists_ = hlp.request("GET", url) self.lists = {} for list_input in lists_: list_input["board"] = self list_input["exempt"] = True if list_input["name"] in EXEMPT else False board_list = List(list_input) self.lists[board_list.name] = board_list print("Fetched lists and cards.")
def test_update(client): response = helpers.request( client, 'PATCH', '/widgets/1?user_id=foo', { 'id': '1', 'owner_id': 'foo', 'name': "Updated", }, ) assert response.status_code == 204
def test_admin_update(client, credentials, result): response = request( client, 'PATCH', '/admin_widgets/2', { 'id': '2', 'name': 'Updated', }, query_string=credentials, ) assert_response(response, result)
def test_create(client, credentials, result): response = request( client, 'POST', '/widgets', { 'name': 'Created', 'tenant_id': TENANT_ID_2, }, query_string=credentials, ) assert_response(response, result)
def test_update(client): response = request( client, 'PATCH', '/widgets/1?user_id=foo', { 'id': '1', 'owner_id': 'foo', 'name': "Updated", }, ) assert_response(response, 204)
def test_relay_cursor_create_sorted(client): response = request( client, 'POST', '/relay_cursor_widgets?sort=size,-id', { 'size': 1, }, ) assert get_meta(response) == { 'cursor': 'MQ.Nw', }
def test_error_update_create_missing_unauthorized(client): response = request( client, 'PUT', '/widgets_create_missing/4?user_id=bar&owner_id=foo', { 'id': '4', 'name': "Created", } ) assert_response(response, 403, [{ 'code': 'invalid_user' }])
def test_relay_cursor_create(client): response = request( client, 'POST', '/relay_cursor_widgets', { 'size': 1, }, ) assert get_meta(response) == { 'cursor': 'Nw', }
def test_error_create_unauthorized(client): response = request( client, 'POST', '/widgets?user_id=bar', { 'owner_id': 'foo', 'name': "Created", }, ) assert_response(response, 403, [{ 'code': 'invalid_user' }])
def _watch_resource_iterator(label, targetFolder, url, method, payload, current, folderAnnotation, resource, logger): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current) if namespace == "ALL": stream = watch.Watch().stream(getattr(v1, _list_for_all_namespaces[resource])) else: stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace) for event in stream: destFolder = targetFolder metadata = event['object'].metadata if metadata.labels is None: continue logger.info("Working on %s %s/%s" % (resource, metadata.namespace, metadata.name)) if label in event['object'].metadata.labels.keys(): logger.info("%s with label found" % resource) if event['object'].metadata.annotations is not None: if folderAnnotation in event['object'].metadata.annotations.keys(): destFolder = event['object'].metadata.annotations[folderAnnotation] logger.info("Found a folder override annotation, placing the %s in: %s" % (resource, destFolder)) dataMap = event['object'].data if dataMap is None: logger.info("%s does not have data." % resource) continue eventType = event['type'] for data_key in dataMap.keys(): logger.info("File in %s %s %s " % (resource, data_key, eventType)) if (eventType == "ADDED") or (eventType == "MODIFIED"): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource, logger) writeTextToFile(destFolder, filename, filedata) if url is not None: request(url, method, logger, payload) else: filename = data_key[:-4] if data_key.endswith(".url") else data_key removeFile(destFolder, filename, logger) if url is not None: request(url, method, logger, payload)
def list_resources(label, label_value, target_folder, request_url, request_method, request_payload, namespace, folder_annotation, resource, unique_filenames, script, enable_5xx): v1 = client.CoreV1Api() # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label additional_args = {'label_selector': label_selector} if namespace != "ALL": additional_args['namespace'] = namespace ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args) files_changed = False # For all the found resources for item in ret.items: metadata = item.metadata print( f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}" ) # Get the destination folder dest_folder = _get_destination_folder(metadata, target_folder, folder_annotation) if resource == RESOURCE_CONFIGMAP: files_changed |= _process_config_map(dest_folder, item, resource, unique_filenames, enable_5xx) else: files_changed = _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx) if script and files_changed: execute(script) if request_url and files_changed: request(request_url, request_method, enable_5xx, request_payload)
def list_resources(label, label_value, target_folder, url, method, payload, current_namespace, folder_annotation, resource, unique_filenames, script): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current_namespace) # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label if namespace == "ALL": ret = getattr( v1, _list_for_all_namespaces[resource])(label_selector=label_selector) else: ret = getattr(v1, _list_namespaced[resource])( namespace=namespace, label_selector=label_selector) files_changed = False # For all the found resources for item in ret.items: metadata = item.metadata print( f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}" ) # Get the destination folder dest_folder = _get_destination_folder(metadata, target_folder, folder_annotation) if resource == RESOURCE_CONFIGMAP: files_changed |= _process_config_map(dest_folder, item, resource, unique_filenames) else: files_changed = _process_secret(dest_folder, item, resource, unique_filenames) if url and files_changed: request(url, method, payload)
def test_commit_conflict(client): response = helpers.request( client, 'POST', '/widgets', { 'name': "Foo", }, ) assert response.status_code == 409 assert helpers.get_errors(response) == [{ 'code': 'invalid_data.conflict', }]
def test_id_missing(client): response = helpers.request( client, 'PATCH', '/widgets/1', { 'name': "Bar", }, ) assert response.status_code == 422 assert helpers.get_errors(response) == [{ 'code': 'invalid_id.missing', }]
def _get_file_data_and_name(full_filename, content, resource): if resource == "secret": file_data = base64.b64decode(content).decode() else: file_data = content if full_filename.endswith(".url"): filename = full_filename[:-4] file_data = request(file_data, "GET").text else: filename = full_filename return filename, file_data
def test_id_forbidden(client): response = helpers.request( client, 'POST', '/widgets', { 'id': '2', 'name': "Bar", }, ) assert response.status_code == 403 assert helpers.get_errors(response) == [{ 'code': 'invalid_id.forbidden', }]
def test_error_missing_id(client): response = request( client, 'PUT', '/children/1', { 'id': '1', 'name': "Updated Child", 'parent': {}, }, ) assert_response(response, 422, [{ 'code': 'invalid_related.missing_id', 'source': {'pointer': '/data/parent'}, }])
def test_error_create_unauthorized(client): response = helpers.request( client, 'POST', '/widgets?user_id=bar', { 'owner_id': 'foo', 'name': "Created", }, ) assert response.status_code == 403 assert helpers.get_errors(response) == [{ 'code': 'invalid_user' }]
def register_app(name, url): # Register Client with Kong # Single API Service data = { 'name': name, 'url': url, } client_info = request(method='post', url=f'{KONG_URL}/services/', data=data) client_id = client_info['id'] # ADD CORS Plugin to Kong for whole domain CORS PLUGIN_URL = f'{KONG_URL}/services/{name}/plugins' data_cors = { 'name': 'cors', 'config.credentials': 'true', 'config.exposed_headers': 'Authorization', 'config.headers': 'Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, Authorization', 'config.max_age': 3600, 'config.methods': ['HEAD', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'], 'config.origins': f'{HOST}/*', } request(method='post', url=PLUGIN_URL, data=data_cors) # Routes # Add a route which we will NOT protect ROUTE_URL = f'{KONG_URL}/services/{name}/routes' data_route = { 'paths': [f'/{name}'], 'strip_path': 'false', 'preserve_host': 'false', # This is keycloak specific. } request(method='post', url=ROUTE_URL, data=data_route) return client_id
def test_error_update_unauthorized(client): not_found_response = request( client, 'PATCH', '/widgets/1?user_id=bar', { 'id': '1', 'owner_id': 'bar', 'name': "Updated", }, ) assert_response(not_found_response, 404) forbidden_response = request( client, 'PATCH', '/widgets/3?user_id=bar', { 'id': '3', 'owner_id': 'bar', 'name': "Updated", }, ) assert_response(forbidden_response, 403, [{ 'code': 'invalid_user' }])
def remove_service(config, realm): def _realm_in_service(realm, service): return any( [path.strip('/').startswith(realm) for path in service['paths']]) name = config['name'] # service name purge = realm == '*' # remove service in ALL realms if purge: print(f'Removing service "{name}" from ALL realms') else: print(f'Removing service "{name}" from realm {realm}') ep_types = [EPT_OIDC, EPT_PUBLIC] for ep_type in ep_types: print(f' Removing "{ep_type}" endpoints') endpoints = config.get(f'{ep_type}_endpoints', []) for ep in endpoints: ep_name = ep['name'] service_name = f'{name}_{ep_type}_{ep_name}' routes_url = f'{KONG_URL}/services/{service_name}/routes' try: res = request(method='get', url=routes_url) for service in res['data']: if purge or _realm_in_service(realm, service): ep_des = f'"{ep_name}" >> {service["paths"]}' try: request(method='delete', url=f'{KONG_URL}/routes/{service["id"]}') print(f' + Removed endpoint {ep_des}') except HTTPError: print(f' - Could not remove endpoint {ep_des}') except HTTPError: print(f' - Route not found "{ep_name}"" at {routes_url}')
def _get_file_data_and_name(full_filename, content, content_type=CONTENT_TYPE_TEXT): if content_type == CONTENT_TYPE_BASE64_BINARY: file_data = base64.b64decode(content) else: file_data = content if full_filename.endswith(".url"): filename = full_filename[:-4] file_data = request(file_data, "GET").text else: filename = full_filename return filename, file_data
def test_create(client): response = request( client, 'POST', '/widgets', { 'name': "Qux", 'description': "qux widget", }, ) assert response.headers['Location'] == 'http://localhost/widgets/4' assert_response(response, 201, { 'id': '4', 'name': "Qux", 'description': "qux widget", })
def _get_file_data_and_name(full_filename, content, resource): if resource == "secret": file_data = base64.b64decode(content).decode() else: file_data = content if full_filename.endswith(".url"): filename = full_filename[:-4] file_data = request(file_data, "GET").text elif full_filename.endswith(".mac"): filename = full_filename[:-4] filename = ":".join(filename[i:i+2] for i in range(0, len(filename), 2)) else: filename = full_filename return filename, file_data
def _get_file_data_and_name(full_filename, content, resource, content_type="ascii"): if resource == "secret": file_data = base64.b64decode(content).decode() elif content_type == "binary": file_data = base64.decodebytes(content.encode('ascii')) else: file_data = content if full_filename.endswith(".url"): filename = full_filename[:-4] file_data = request(file_data, "GET").text else: filename = full_filename return filename, file_data
def is_alive(self): logging.main_logger.debug("[region] 'is_alive' called") import sim self.alive = False mysim = sim.Sim(self.sim_path, True) # check if the sim is responding to simstatus check_monitorstats = False from helpers import request url = 'http://127.0.0.1:' + mysim.port + '/monitorstats/' + self.region_uuid logging.main_logger.debug('Url called : %s' % (url)) req = request(url) if req: # check if status is 200 if req.getcode() == 200: check_monitorstats = True self.alive = True return self.alive
def listResources(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource, uniqueFilenames): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", currentNamespace) # Filter resources based on label and value or just label labelSelector = f"{label}={labelValue}" if labelValue else label if namespace == "ALL": ret = getattr( v1, _list_for_all_namespaces[resource])(label_selector=labelSelector) else: ret = getattr(v1, _list_namespaced[resource])(namespace=namespace, label_selector=labelSelector) files_changed = False # For all the found resources for sec in ret.items: metadata = sec.metadata print( f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}" ) # Get the destination folder destFolder = _get_destination_folder(metadata, targetFolder, folderAnnotation) # Check if it's an empty ConfigMap or Secret if resource == "configmap": if sec.data is None and sec.binary_data is None: print(f"{timestamp()} No data/binaryData field in {resource}") continue else: if sec.data is None: print(f"{timestamp()} No data field in {resource}") continue # Each key on the data is a file if sec.data is not None: for data_key in sec.data.keys(): filename, filedata = _get_file_data_and_name( data_key, sec.data[data_key], resource) if uniqueFilenames: filename = uniqueFilename(filename=filename, namespace=metadata.namespace, resource=resource, resource_name=metadata.name) files_changed |= writeTextToFile(destFolder, filename, filedata) # Each key on the binaryData is a file if resource == "configmap" and sec.binary_data is not None: for data_key in sec.binary_data.keys(): filename, filedata = _get_file_data_and_name( data_key, sec.binary_data[data_key], resource, content_type="binary") if uniqueFilenames: filename = uniqueFilename(filename=filename, namespace=metadata.namespace, resource=resource, resource_name=metadata.name) files_changed |= writeTextToFile(destFolder, filename, filedata, data_type="binary") if url and files_changed: request(url, method, payload)