def listResources(label, targetFolder, url, method, payload, current, folderAnnotation, resource, logger):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", current)
if namespace == "ALL":
ret = getattr(v1, _list_for_all_namespaces[resource])()
else:
ret = getattr(v1, _list_namespaced[resource])(namespace=namespace)
for sec in ret.items:
destFolder = targetFolder
metadata = sec.metadata
if metadata.labels is None:
continue
logger.info("Working on %s: %s/%s" % (resource, metadata.namespace, metadata.name))
if label in sec.metadata.labels.keys():
logger.info("Found %s with label" % resource)
if sec.metadata.annotations is not None:
if folderAnnotation in sec.metadata.annotations.keys():
destFolder = sec.metadata.annotations[folderAnnotation]
dataMap = sec.data
if dataMap is None:
logger.info("No data field in %s" % resource)
continue
if label in sec.metadata.labels.keys():
for data_key in dataMap.keys():
filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
resource, logger)
writeTextToFile(destFolder, filename, filedata)
if url is not None:
request(url, method, logger, payload)
def generate_advert_county_urls():
"""
Generator function navigates through the pages of real estate adverts
and extracts all the links to the adverts.
"""
# Build the BS object from the page source
page_source = None
while not page_source:
page_source = helpers.request(_BASE_URL, proxies=False)
if page_source.status_code != 200:
page_source = None
page_source = page_source.text.encode('utf-8')
soup = BeautifulSoup(page_source, 'lxml')
# Extract the links to the offers filtered by provider
for provider_url in soup.findAll('a', {'class': 'fleft tab tdnone topTabOffer'}):
provider_url = provider_url['href']
provider_page_source = helpers.request(provider_url, proxies=False)
if not provider_page_source:
continue
provider_page_source = provider_page_source.text.encode('utf-8')
provider_soup = BeautifulSoup(provider_page_source, 'lxml')
for county_url in provider_soup.select('div.locationlinks.margintop10 li a'):
if county_url.has_attr('href'):
yield county_url['href']
def test_error_update_unauthorized(client):
not_found_response = helpers.request(
client,
'PATCH', '/widgets/1?user_id=bar',
{
'id': '1',
'owner_id': 'bar',
'name': "Updated",
},
)
assert not_found_response.status_code == 404
forbidden_response = helpers.request(
client,
'PATCH', '/widgets/3?user_id=bar',
{
'id': '3',
'owner_id': 'bar',
'name': "Updated",
},
)
assert forbidden_response.status_code == 403
assert helpers.get_errors(forbidden_response) == [{
'code': 'invalid_user'
}]
def listResources(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", currentNamespace)
# Filter resources based on label and value or just label
labelSelector=f"{label}={labelValue}" if labelValue else label
if namespace == "ALL":
ret = getattr(v1, _list_for_all_namespaces[resource])(label_selector=labelSelector)
else:
ret = getattr(v1, _list_namespaced[resource])(namespace=namespace, label_selector=labelSelector)
# For all the found resources
for sec in ret.items:
metadata = sec.metadata
print(f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}")
# Get the destination folder
destFolder = _get_destination_folder(metadata, targetFolder, folderAnnotation)
# Check if it's an empty ConfigMap or Secret
dataMap = sec.data
if dataMap is None:
print(f"{timestamp()} No data field in {resource}")
continue
# Each key on the data is a file
for data_key in dataMap.keys():
filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
resource)
writeTextToFile(destFolder, filename, filedata)
if url:
request(url, method, payload)
def remove_stats(self, *args):
url = f"https://api.trello.com/1/cards/{self.id}"
for stat in args:
try:
del self.stats[stat]
except KeyError:
continue
self.desc = hlp.format_desc(self.stats)
hlp.request("PUT", url, desc=self.desc)
def add_service(config, realm):
name = config['name'] # service name
host = config['host'] # service host
print(f'\nAdding realm "{realm}" to service "{name}"')
# OIDC plugin settings (same for all OIDC endpoints)
oidc_data = _get_service_oidc_payload(name, realm)
ep_types = [EPT_PUBLIC, EPT_OIDC]
for ep_type in ep_types:
print(f' Adding "{ep_type}" endpoints')
endpoints = config.get(f'{ep_type}_endpoints', [])
for ep in endpoints:
ep_name = ep['name']
ep_url = ep['url']
service_name = f'{name}_{ep_type}_{ep_name}'
data = {
'name': service_name,
'url': f'{host}{ep_url}',
}
try:
request(method='post', url=f'{KONG_URL}/services/', data=data)
print(f' + Added service "{ep_name}"')
except HTTPError:
print(f' - Could not add service "{ep_name}"')
ROUTE_URL = f'{KONG_URL}/services/{service_name}/routes'
path = ep.get('route_path') or f'/{realm}/{name}{ep_url}'
route_data = {
'paths': [
path,
],
'strip_path': ep.get('strip_path', 'false'),
}
try:
route_info = request(method='post',
url=ROUTE_URL,
data=route_data)
# OIDC routes are protected using the "kong-oidc-auth" plugin
if ep_type == EPT_OIDC:
protected_route_id = route_info['id']
request(
method='post',
url=f'{KONG_URL}/routes/{protected_route_id}/plugins',
data=oidc_data,
)
print(f' + Added route {ep_url} >> {path}')
except HTTPError:
print(f' - Could not add route {ep_url} >> {path}')
print(f'Service "{name}" now being served by kong for realm "{realm}"\n')
def archive_cards(self):
url = f"https://api.trello.com/1/lists/{self.id}/archiveAllCards"
hlp.request("POST", url)
card_names = [card.name for card in self.cards]
print(f"All cards archived in list {self.name}: {card_names}")
self._log_date()
self.board._update_task_file()
self.cards = []
self.board._get_cards()
def _watch_resource_iterator(label, labelValue, targetFolder, url, method, payload,
currentNamespace, folderAnnotation, resource, uniqueFilenames):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", currentNamespace)
# Filter resources based on label and value or just label
labelSelector=f"{label}={labelValue}" if labelValue else label
if namespace == "ALL":
stream = watch.Watch().stream(getattr(v1, _list_for_all_namespaces[resource]), label_selector=labelSelector)
else:
stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace, label_selector=labelSelector)
# Process events
for event in stream:
metadata = event["object"].metadata
print(f"{timestamp()} Working on {resource} {metadata.namespace}/{metadata.name}")
# Get the destination folder
destFolder = _get_destination_folder(metadata, targetFolder, folderAnnotation)
# Check if it's an empty ConfigMap or Secret
dataMap = event["object"].data
if dataMap is None:
print(f"{timestamp()} {resource} does not have data.")
continue
eventType = event["type"]
# Each key on the data is a file
for data_key in dataMap.keys():
print(f"{timestamp()} File in {resource} {data_key} {eventType}")
if (eventType == "ADDED") or (eventType == "MODIFIED"):
filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
resource)
if uniqueFilenames:
filename = uniqueFilename(filename = filename,
namespace = metadata.namespace,
resource = resource,
resource_name = metadata.name)
writeTextToFile(destFolder, filename, filedata)
else:
# Get filename from event
filename = data_key[:-4] if data_key.endswith(".url") else data_key
if uniqueFilenames:
filename = uniqueFilename(filename = filename,
namespace = metadata.namespace,
resource = resource,
resource_name = metadata.name)
removeFile(destFolder, filename)
if url:
request(url, method, payload)
def sort_list(self):
prefer_order = lambda x: (
x.stats.get("priority", 999),
x.labels[0]["name"],
x.due,
x.name)
self.cards = sorted(self.cards, key=prefer_order)
for rank, card in enumerate(self.cards, start=1):
url = f"https://api.trello.com/1/cards/{card.id}"
hlp.request("PUT", url, pos=rank)
def list_resources(label, label_value, target_folder, request_url,
request_method, request_payload, namespace,
folder_annotation, resource, unique_filenames, script,
enable_5xx, ignore_already_processed):
v1 = client.CoreV1Api()
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
additional_args = {'label_selector': label_selector}
if namespace != "ALL":
additional_args['namespace'] = namespace
ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args)
files_changed = False
# For all the found resources
for item in ret.items:
metadata = item.metadata
# Ignore already processed resource
# Avoid numerous logs about useless resource processing each time the LIST loop reconnects
if ignore_already_processed:
if _resources_version_map.get(
metadata.namespace +
metadata.name) == metadata.resource_version:
# print(f"{timestamp()} Ignoring {resource} {metadata.namespace}/{metadata.name}")
continue
_resources_version_map[metadata.namespace +
metadata.name] = metadata.resource_version
print(
f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}"
)
# Get the destination folder
dest_folder = _get_destination_folder(metadata, target_folder,
folder_annotation)
if resource == RESOURCE_CONFIGMAP:
files_changed |= _process_config_map(dest_folder, item, resource,
unique_filenames, enable_5xx)
else:
files_changed = _process_secret(dest_folder, item, resource,
unique_filenames, enable_5xx)
if script and files_changed:
execute(script)
if request_url and files_changed:
request(request_url, request_method, enable_5xx, request_payload)
def _watch_resource_iterator(label, label_value, target_folder, url, method,
payload, current_namespace, folder_annotation,
resource, unique_filenames, script):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", current_namespace)
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
if namespace == "ALL":
stream = watch.Watch().stream(getattr(
v1, _list_for_all_namespaces[resource]),
label_selector=label_selector,
_request_timeout=60)
else:
stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]),
namespace=namespace,
label_selector=label_selector,
_request_timeout=60)
# Process events
for event in stream:
item = event["object"]
metadata = item.metadata
event_type = event["type"]
print(
f"{timestamp()} Working on {event_type} {resource} {metadata.namespace}/{metadata.name}"
)
files_changed = False
# Get the destination folder
dest_folder = _get_destination_folder(metadata, target_folder,
folder_annotation)
item_removed = event_type == "DELETED"
if resource == RESOURCE_CONFIGMAP:
files_changed |= _process_config_map(dest_folder, item, resource,
unique_filenames,
item_removed)
else:
files_changed |= _process_secret(dest_folder, item, resource,
unique_filenames, item_removed)
if script and files_changed:
execute(script)
if url and files_changed:
request(url, method, payload)
def create_card(self):
self.assign_due_date()
self.create_card_body()
inbox = self._board.lists["Inbox"]
url = "https://api.trello.com/1/cards"
params = {
"idList": inbox.id,
"name": self.name,
"idLabels": self._label_ids,
"due": self.due,
"desc": hlp.format_desc(self.card_body)
}
hlp.request("POST", url, **params)
print(f"Posted card: {self.name} (due {self.due.date()})")
def test_resource_rules(app, views, client):
api = Api(app)
api.add_resource(
base_rule='/widget/<id>',
base_view=views['widget'],
alternate_rule='/widgets',
alternate_view=views['widget_list'],
)
get_response = client.get('/widget/1')
assert get_response.status_code == 200
assert helpers.get_data(get_response) == {
'id': '1',
}
post_response = helpers.request(
client,
'POST', '/widgets',
{},
)
assert post_response.status_code == 201
assert post_response.headers['Location'] == 'http://localhost/widget/2'
assert helpers.get_data(post_response) == {
'id': '2',
}
def _get_labels(self):
url = f"https://api.trello.com/1/boards/{self.board_id}/labels/"
labels = hlp.request("GET", url)
label_map = {label["name"]: label["id"] for label in labels}
self.labels = label_map
self.label_names = {id: name for name, id in self.labels.items()}
print("Fetched labels")
def test_many(client):
response = request(
client,
'PUT', '/parents/1',
{
'id': '1',
'name': "Updated Parent",
'children': [
{'id': '1'},
{'id': '2'},
],
},
)
assert_response(response, 200, {
'id': '1',
'name': "Updated Parent",
'children': [
{
'id': '1',
'name': "Child 1",
},
{
'id': '2',
'name': "Child 2",
},
],
})
def test_many(client):
response = helpers.request(
client,
'PUT', '/parents/1',
{
'id': '1',
'name': "Updated Parent",
'children': [
{'id': '1'},
{'id': '2'},
],
},
)
assert helpers.get_data(response) == {
'id': '1',
'name': "Updated Parent",
'children': [
{
'id': '1',
'name': "Child 1",
},
{
'id': '2',
'name': "Child 2",
},
],
}
def test_many_nested(client):
response = request(
client,
'PUT', '/nested_parents/1',
{
'id': '1',
'name': "Updated Parent",
'children': [
{'name': "Child 3"},
{'name': "Child 4"},
],
},
)
assert_response(response, 200, {
'id': '1',
'name': "Updated Parent",
'children': [
{
'id': '3',
'name': "Child 3",
},
{
'id': '4',
'name': "Child 4",
},
],
})
def generate_advert_urls():
"""
Generate all the adverd offer urls from all the indexed pages within
the real estate 'super category' of the webside.
"""
for category_url in generate_advert_category_urls():
url_template = category_url + '&page={}'
for index in itertools.count(1):
# Access the page and get the source
page_url = url_template.format(index)
page_source = helpers.request(page_url, proxies=False)
page_source = page_source.text.encode('utf-8')
page_soup = BeautifulSoup(page_source, 'lxml')
try:
# Extract urls from the page
for offer_url in page_soup.select('td.offer a.marginright5.link'):
if offer_url.has_attr('href') and offer_url['href'] not in _EXTRACTED_OFFER_URLS:
_EXTRACTED_OFFER_URLS.append(offer_url['href'])
yield offer_url['href']
# Look exit condition
last_page = page_soup.select('a.link.pageNextPrev')
if (len(last_page) < 2 and index > 1) or not last_page:
break
except Exception as e:
print(e.with_traceback(e.__traceback__))
with open('urls_error.htm', 'wt') as urls_error:
print('an error has occurred')
urls_error.write(page_soup.prettify())
break
def call(self):
# Build the URL for the API (they are RESTful)
kvFormat = '{}={}'
params = ''
# Now turn JSON like ``"a":"b"`` into URL parameters like ``a=b``
if self._payload is not {}:
params = '&'.join([
kvFormat.format(key, str(self._payload[key]))
for key in sorted(self._payload.keys())
])
url = self.__host % (self._path, params)
else:
url = self.__host_no_params
# Obtain the request signature
timestamp = str(int((helpers.time.time() + helpers.timeOffset) * 1000))
signature = signer.sign(self._path, timestamp, params)
# Generate the headers, fill with the request signature
headers = {
'KC-API-KEY': signer.key,
'KC-API-NONCE': timestamp,
'KC-API-SIGNATURE': signature,
'Content-Type': 'application/x-www-form-urlencoded'
}
# Pass the actual request off to helpers, since they do error checking and more
return helpers.request(self._method, url, headers, self)
def is_alive(self):
logging.main_logger.debug("[sim] 'is_alive' called")
self.alive = False
# check if there is a pid file
check_pid = False
if self.pid_file != '':
check_pid = os.path.isfile(self.pid_file)
# check if the sim is responding to simstatus
check_simstatus = False
from helpers import request
req = request('http://127.0.0.1:' + str(self.port) + '/simstatus')
if req:
# check if status is 200
if req.getcode() == 200:
if req.read() == 'OK':
check_simstatus = True
if check_pid and check_simstatus:
self.alive = True
logging.main_logger.debug("[sim] sim %s is alive" % (self.path))
return True
else:
logging.main_logger.debug("[sim] sim %s is not alive" % (self.path))
return False
def _watch_resource_iterator(label, targetFolder, url, method, payload,
current, folderAnnotation, resource):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", current)
if namespace == "ALL":
stream = watch.Watch().stream(
getattr(v1, _list_for_all_namespaces[resource]))
else:
stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]),
namespace=namespace)
for event in stream:
destFolder = targetFolder
metadata = event['object'].metadata
if metadata.labels is None:
continue
print(f'Working on {resource} {metadata.namespace}/{metadata.name}')
if label in event['object'].metadata.labels.keys():
print(f"{resource} with label found")
if event['object'].metadata.annotations is not None:
if folderAnnotation in event[
'object'].metadata.annotations.keys():
destFolder = event['object'].metadata.annotations[
folderAnnotation]
print('Found a folder override annotation, '
f'placing the {resource} in: {destFolder}')
dataMap = event['object'].data
if dataMap is None:
print(f"{resource} does not have data.")
continue
eventType = event['type']
for data_key in dataMap.keys():
print(f"File in {resource} {data_key} {eventType}")
if (eventType == "ADDED") or (eventType == "MODIFIED"):
filename, filedata = _get_file_data_and_name(
data_key, dataMap[data_key], resource)
writeTextToFile(destFolder, filename, filedata)
if url is not None:
request(url, method, payload)
else:
filename = data_key[:-4] if data_key.endswith(
".url") else data_key
removeFile(destFolder, filename)
if url is not None:
request(url, method, payload)
def _watch_resource_iterator(label, label_value, target_folder, request_url,
request_method, request_payload, namespace,
folder_annotation, resource, unique_filenames,
script, enable_5xx):
v1 = client.CoreV1Api()
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
additional_args = {'label_selector': label_selector}
if namespace != "ALL":
additional_args['namespace'] = namespace
stream = watch.Watch().stream(
getattr(v1, _list_namespace[namespace][resource]), **additional_args)
# Process events
for event in stream:
item = event["object"]
metadata = item.metadata
event_type = event["type"]
print(
f"{timestamp()} Working on {event_type} {resource} {metadata.namespace}/{metadata.name}"
)
files_changed = False
# Get the destination folder
dest_folder = _get_destination_folder(metadata, target_folder,
folder_annotation)
item_removed = event_type == "DELETED"
if resource == RESOURCE_CONFIGMAP:
files_changed |= _process_config_map(dest_folder, item, resource,
unique_filenames, enable_5xx,
item_removed)
else:
files_changed |= _process_secret(dest_folder, item, resource,
unique_filenames, enable_5xx,
item_removed)
if script and files_changed:
execute(script)
if request_url and files_changed:
request(request_url, request_method, enable_5xx, request_payload)
def test_update(client):
update_response = helpers.request(client, "PATCH", "/widgets/1/2", {"id_1": "1", "id_2": "2", "name": "Qux"})
assert update_response.status_code == 204
retrieve_response = client.get("/widgets/1/2")
assert retrieve_response.status_code == 200
assert helpers.get_data(retrieve_response) == {"id_1": "1", "id_2": "2", "name": "Qux"}
def move_card(self, new_list_):
url = f"https://api.trello.com/1/cards/{self.id}"
new_list = self.list.board.lists[new_list_]
r = hlp.request("PUT", url, idList=new_list.id)
if r.status_code == 200:
self.list.cards.remove(self)
self.list = new_list
self.list.cards.append(self)
print(f"Moved card {self.name} to {self.list.name} (due {self.due})")
def test_create(client):
response = helpers.request(
client,
'POST', '/widgets?user_id=foo',
{
'owner_id': 'foo',
'name': "Created",
},
)
assert response.status_code == 201
def test_update_update_missing(client):
response = request(
client,
'PUT', '/widgets_create_missing/4?user_id=foo&owner_id=foo',
{
'id': '4',
'name': "Created",
}
)
assert_response(response, 204)
def test_limit_offset_create(client):
response = request(
client,
'POST', '/limit_offset_widgets',
{
'size': 1,
},
)
assert 'meta' not in get_body(response)
def get_list_cards(self):
self.cards = []
url = f'https://api.trello.com/1/lists/{self.id}/cards'
fields = ["id","name","desc","due","labels"]
card_list = hlp.request("GET", url, fields=fields)
for card_input in card_list:
card_input["due"] = hlp.localize_ts(card_input["due"])
card_input["list"] = self
self.cards.append(Card(card_input))
self.time_sum()
def test_create(client):
response = request(
client,
'POST', '/widgets?user_id=foo',
{
'owner_id': 'foo',
'name': "Created",
},
)
assert_response(response, 201)
def test_page_create(client):
response = request(
client,
'POST', '/page_widgets',
{
'size': 1,
},
)
assert 'meta' not in get_body(response)
def test_update_tenant_id(client, credentials, result):
response = request(
client,
'PATCH', '/widgets/2',
{
'id': '2',
'tenant_id': TENANT_ID_1,
},
query_string=credentials,
)
assert_response(response, result)
def get_checklists(self):
self.checklists = defaultdict(list)
url = f"https://api.trello.com/1/cards/{self.id}/checklists"
raw = hlp.request("GET", url)
for checklist in raw:
name = checklist["name"]
check_items = sorted(checklist["checkItems"], key=lambda x: x["pos"])
for checkitem in check_items:
item_name = checkitem["name"]
self.checklists[name].append(item_name)
def _get_lists(self):
url = f"https://api.trello.com/1/boards/{self.board_id}/lists/"
lists_ = hlp.request("GET", url)
self.lists = {}
for list_input in lists_:
list_input["board"] = self
list_input["exempt"] = True if list_input["name"] in EXEMPT else False
board_list = List(list_input)
self.lists[board_list.name] = board_list
print("Fetched lists and cards.")
def test_update(client):
response = helpers.request(
client,
'PATCH', '/widgets/1?user_id=foo',
{
'id': '1',
'owner_id': 'foo',
'name': "Updated",
},
)
assert response.status_code == 204
def test_admin_update(client, credentials, result):
response = request(
client,
'PATCH', '/admin_widgets/2',
{
'id': '2',
'name': 'Updated',
},
query_string=credentials,
)
assert_response(response, result)
def test_create(client, credentials, result):
response = request(
client,
'POST', '/widgets',
{
'name': 'Created',
'tenant_id': TENANT_ID_2,
},
query_string=credentials,
)
assert_response(response, result)
def test_update(client):
response = request(
client,
'PATCH', '/widgets/1?user_id=foo',
{
'id': '1',
'owner_id': 'foo',
'name': "Updated",
},
)
assert_response(response, 204)
def test_relay_cursor_create_sorted(client):
response = request(
client,
'POST', '/relay_cursor_widgets?sort=size,-id',
{
'size': 1,
},
)
assert get_meta(response) == {
'cursor': 'MQ.Nw',
}
def test_error_update_create_missing_unauthorized(client):
response = request(
client,
'PUT', '/widgets_create_missing/4?user_id=bar&owner_id=foo',
{
'id': '4',
'name': "Created",
}
)
assert_response(response, 403, [{
'code': 'invalid_user'
}])
def test_relay_cursor_create(client):
response = request(
client,
'POST', '/relay_cursor_widgets',
{
'size': 1,
},
)
assert get_meta(response) == {
'cursor': 'Nw',
}
def test_error_create_unauthorized(client):
response = request(
client,
'POST', '/widgets?user_id=bar',
{
'owner_id': 'foo',
'name': "Created",
},
)
assert_response(response, 403, [{
'code': 'invalid_user'
}])
def _watch_resource_iterator(label, targetFolder, url, method, payload,
current, folderAnnotation, resource, logger):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", current)
if namespace == "ALL":
stream = watch.Watch().stream(getattr(v1, _list_for_all_namespaces[resource]))
else:
stream = watch.Watch().stream(getattr(v1, _list_namespaced[resource]), namespace=namespace)
for event in stream:
destFolder = targetFolder
metadata = event['object'].metadata
if metadata.labels is None:
continue
logger.info("Working on %s %s/%s" % (resource, metadata.namespace, metadata.name))
if label in event['object'].metadata.labels.keys():
logger.info("%s with label found" % resource)
if event['object'].metadata.annotations is not None:
if folderAnnotation in event['object'].metadata.annotations.keys():
destFolder = event['object'].metadata.annotations[folderAnnotation]
logger.info("Found a folder override annotation, placing the %s in: %s" % (resource, destFolder))
dataMap = event['object'].data
if dataMap is None:
logger.info("%s does not have data." % resource)
continue
eventType = event['type']
for data_key in dataMap.keys():
logger.info("File in %s %s %s " % (resource, data_key, eventType))
if (eventType == "ADDED") or (eventType == "MODIFIED"):
filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
resource, logger)
writeTextToFile(destFolder, filename, filedata)
if url is not None:
request(url, method, logger, payload)
else:
filename = data_key[:-4] if data_key.endswith(".url") else data_key
removeFile(destFolder, filename, logger)
if url is not None:
request(url, method, logger, payload)
def list_resources(label, label_value, target_folder, request_url,
request_method, request_payload, namespace,
folder_annotation, resource, unique_filenames, script,
enable_5xx):
v1 = client.CoreV1Api()
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
additional_args = {'label_selector': label_selector}
if namespace != "ALL":
additional_args['namespace'] = namespace
ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args)
files_changed = False
# For all the found resources
for item in ret.items:
metadata = item.metadata
print(
f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}"
)
# Get the destination folder
dest_folder = _get_destination_folder(metadata, target_folder,
folder_annotation)
if resource == RESOURCE_CONFIGMAP:
files_changed |= _process_config_map(dest_folder, item, resource,
unique_filenames, enable_5xx)
else:
files_changed = _process_secret(dest_folder, item, resource,
unique_filenames, enable_5xx)
if script and files_changed:
execute(script)
if request_url and files_changed:
request(request_url, request_method, enable_5xx, request_payload)
def list_resources(label, label_value, target_folder, url, method, payload,
current_namespace, folder_annotation, resource,
unique_filenames, script):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", current_namespace)
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
if namespace == "ALL":
ret = getattr(
v1,
_list_for_all_namespaces[resource])(label_selector=label_selector)
else:
ret = getattr(v1, _list_namespaced[resource])(
namespace=namespace, label_selector=label_selector)
files_changed = False
# For all the found resources
for item in ret.items:
metadata = item.metadata
print(
f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}"
)
# Get the destination folder
dest_folder = _get_destination_folder(metadata, target_folder,
folder_annotation)
if resource == RESOURCE_CONFIGMAP:
files_changed |= _process_config_map(dest_folder, item, resource,
unique_filenames)
else:
files_changed = _process_secret(dest_folder, item, resource,
unique_filenames)
if url and files_changed:
request(url, method, payload)
def test_commit_conflict(client):
response = helpers.request(
client,
'POST', '/widgets',
{
'name': "Foo",
},
)
assert response.status_code == 409
assert helpers.get_errors(response) == [{
'code': 'invalid_data.conflict',
}]
def test_id_missing(client):
response = helpers.request(
client,
'PATCH', '/widgets/1',
{
'name': "Bar",
},
)
assert response.status_code == 422
assert helpers.get_errors(response) == [{
'code': 'invalid_id.missing',
}]
def _get_file_data_and_name(full_filename, content, resource):
if resource == "secret":
file_data = base64.b64decode(content).decode()
else:
file_data = content
if full_filename.endswith(".url"):
filename = full_filename[:-4]
file_data = request(file_data, "GET").text
else:
filename = full_filename
return filename, file_data
def test_id_forbidden(client):
response = helpers.request(
client,
'POST', '/widgets',
{
'id': '2',
'name': "Bar",
},
)
assert response.status_code == 403
assert helpers.get_errors(response) == [{
'code': 'invalid_id.forbidden',
}]
def test_error_missing_id(client):
response = request(
client,
'PUT', '/children/1',
{
'id': '1',
'name': "Updated Child",
'parent': {},
},
)
assert_response(response, 422, [{
'code': 'invalid_related.missing_id',
'source': {'pointer': '/data/parent'},
}])
def test_error_create_unauthorized(client):
response = helpers.request(
client,
'POST', '/widgets?user_id=bar',
{
'owner_id': 'foo',
'name': "Created",
},
)
assert response.status_code == 403
assert helpers.get_errors(response) == [{
'code': 'invalid_user'
}]
def register_app(name, url):
# Register Client with Kong
# Single API Service
data = {
'name': name,
'url': url,
}
client_info = request(method='post',
url=f'{KONG_URL}/services/',
data=data)
client_id = client_info['id']
# ADD CORS Plugin to Kong for whole domain CORS
PLUGIN_URL = f'{KONG_URL}/services/{name}/plugins'
data_cors = {
'name': 'cors',
'config.credentials': 'true',
'config.exposed_headers': 'Authorization',
'config.headers':
'Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, Authorization',
'config.max_age': 3600,
'config.methods': ['HEAD', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'],
'config.origins': f'{HOST}/*',
}
request(method='post', url=PLUGIN_URL, data=data_cors)
# Routes
# Add a route which we will NOT protect
ROUTE_URL = f'{KONG_URL}/services/{name}/routes'
data_route = {
'paths': [f'/{name}'],
'strip_path': 'false',
'preserve_host': 'false', # This is keycloak specific.
}
request(method='post', url=ROUTE_URL, data=data_route)
return client_id
def test_error_update_unauthorized(client):
not_found_response = request(
client,
'PATCH', '/widgets/1?user_id=bar',
{
'id': '1',
'owner_id': 'bar',
'name': "Updated",
},
)
assert_response(not_found_response, 404)
forbidden_response = request(
client,
'PATCH', '/widgets/3?user_id=bar',
{
'id': '3',
'owner_id': 'bar',
'name': "Updated",
},
)
assert_response(forbidden_response, 403, [{
'code': 'invalid_user'
}])
def remove_service(config, realm):
def _realm_in_service(realm, service):
return any(
[path.strip('/').startswith(realm) for path in service['paths']])
name = config['name'] # service name
purge = realm == '*' # remove service in ALL realms
if purge:
print(f'Removing service "{name}" from ALL realms')
else:
print(f'Removing service "{name}" from realm {realm}')
ep_types = [EPT_OIDC, EPT_PUBLIC]
for ep_type in ep_types:
print(f' Removing "{ep_type}" endpoints')
endpoints = config.get(f'{ep_type}_endpoints', [])
for ep in endpoints:
ep_name = ep['name']
service_name = f'{name}_{ep_type}_{ep_name}'
routes_url = f'{KONG_URL}/services/{service_name}/routes'
try:
res = request(method='get', url=routes_url)
for service in res['data']:
if purge or _realm_in_service(realm, service):
ep_des = f'"{ep_name}" >> {service["paths"]}'
try:
request(method='delete',
url=f'{KONG_URL}/routes/{service["id"]}')
print(f' + Removed endpoint {ep_des}')
except HTTPError:
print(f' - Could not remove endpoint {ep_des}')
except HTTPError:
print(f' - Route not found "{ep_name}"" at {routes_url}')
def _get_file_data_and_name(full_filename,
content,
content_type=CONTENT_TYPE_TEXT):
if content_type == CONTENT_TYPE_BASE64_BINARY:
file_data = base64.b64decode(content)
else:
file_data = content
if full_filename.endswith(".url"):
filename = full_filename[:-4]
file_data = request(file_data, "GET").text
else:
filename = full_filename
return filename, file_data
def test_create(client):
response = request(
client,
'POST', '/widgets',
{
'name': "Qux",
'description': "qux widget",
},
)
assert response.headers['Location'] == 'http://localhost/widgets/4'
assert_response(response, 201, {
'id': '4',
'name': "Qux",
'description': "qux widget",
})
def _get_file_data_and_name(full_filename, content, resource):
if resource == "secret":
file_data = base64.b64decode(content).decode()
else:
file_data = content
if full_filename.endswith(".url"):
filename = full_filename[:-4]
file_data = request(file_data, "GET").text
elif full_filename.endswith(".mac"):
filename = full_filename[:-4]
filename = ":".join(filename[i:i+2] for i in range(0, len(filename), 2))
else:
filename = full_filename
return filename, file_data
def _get_file_data_and_name(full_filename,
content,
resource,
content_type="ascii"):
if resource == "secret":
file_data = base64.b64decode(content).decode()
elif content_type == "binary":
file_data = base64.decodebytes(content.encode('ascii'))
else:
file_data = content
if full_filename.endswith(".url"):
filename = full_filename[:-4]
file_data = request(file_data, "GET").text
else:
filename = full_filename
return filename, file_data
def is_alive(self):
logging.main_logger.debug("[region] 'is_alive' called")
import sim
self.alive = False
mysim = sim.Sim(self.sim_path, True)
# check if the sim is responding to simstatus
check_monitorstats = False
from helpers import request
url = 'http://127.0.0.1:' + mysim.port + '/monitorstats/' + self.region_uuid
logging.main_logger.debug('Url called : %s' % (url))
req = request(url)
if req:
# check if status is 200
if req.getcode() == 200:
check_monitorstats = True
self.alive = True
return self.alive
def listResources(label, labelValue, targetFolder, url, method, payload,
currentNamespace, folderAnnotation, resource,
uniqueFilenames):
v1 = client.CoreV1Api()
namespace = os.getenv("NAMESPACE", currentNamespace)
# Filter resources based on label and value or just label
labelSelector = f"{label}={labelValue}" if labelValue else label
if namespace == "ALL":
ret = getattr(
v1,
_list_for_all_namespaces[resource])(label_selector=labelSelector)
else:
ret = getattr(v1,
_list_namespaced[resource])(namespace=namespace,
label_selector=labelSelector)
files_changed = False
# For all the found resources
for sec in ret.items:
metadata = sec.metadata
print(
f"{timestamp()} Working on {resource}: {metadata.namespace}/{metadata.name}"
)
# Get the destination folder
destFolder = _get_destination_folder(metadata, targetFolder,
folderAnnotation)
# Check if it's an empty ConfigMap or Secret
if resource == "configmap":
if sec.data is None and sec.binary_data is None:
print(f"{timestamp()} No data/binaryData field in {resource}")
continue
else:
if sec.data is None:
print(f"{timestamp()} No data field in {resource}")
continue
# Each key on the data is a file
if sec.data is not None:
for data_key in sec.data.keys():
filename, filedata = _get_file_data_and_name(
data_key, sec.data[data_key], resource)
if uniqueFilenames:
filename = uniqueFilename(filename=filename,
namespace=metadata.namespace,
resource=resource,
resource_name=metadata.name)
files_changed |= writeTextToFile(destFolder, filename,
filedata)
# Each key on the binaryData is a file
if resource == "configmap" and sec.binary_data is not None:
for data_key in sec.binary_data.keys():
filename, filedata = _get_file_data_and_name(
data_key,
sec.binary_data[data_key],
resource,
content_type="binary")
if uniqueFilenames:
filename = uniqueFilename(filename=filename,
namespace=metadata.namespace,
resource=resource,
resource_name=metadata.name)
files_changed |= writeTextToFile(destFolder,
filename,
filedata,
data_type="binary")
if url and files_changed:
request(url, method, payload)
