def _beautify(data, *, colors: bool, table: bool) -> str:
"""
1. Returns table if `table=True`
1. Returns colored JSON if `json=True`
1. Returns plain JSON otherwise.
"""
if table:
# one dict
if isinstance(data, dict):
data = FlatDict(data, delimiter='.').items()
return tabulate(data, headers=('key', 'value'), tablefmt='fancy_grid')
# list of dicts
if isinstance(data, list) and data and isinstance(data[0], dict):
table = []
for row in data:
row = FlatDict(row, delimiter='.')
keys = tuple(row)
row = [v for _, v in sorted(row.items())]
table.append(row)
return tabulate(table, headers=keys, tablefmt='fancy_grid')
json_params = dict(indent=2, sort_keys=True, ensure_ascii=False)
dumped = json.dumps(data, **json_params)
if not colors:
return dumped
return highlight(dumped, lexers.JsonLexer(), formatters.TerminalFormatter())
def error_422_handler(exc, req, resp, params):
update_content_type(req, resp)
resp.status = exc.status
if req.api_version == '1.0':
exc_data = {
'title': exc.title,
'description': _('Field value error'),
'code': getattr(exc, 'code') or 'entity_error'
}
if hasattr(exc, 'errors'):
exc_data['errors'] = exc.errors
result = ErrorSchema().dump(exc_data)
resp.text = json.dumps(result, cls=LazyEncoder)
else:
_exc_code = exc.status.lower().replace(' ', '_')
_errors = []
if hasattr(exc, 'errors'):
flat = FlatDict(exc.errors, delimiter='/')
for field, errors in flat.items():
if not isinstance(errors, list):
errors = [
str(errors),
]
for title in errors:
_error = {
'id': uuid4(),
'title': _('Field error'),
'detail': _(title),
'status': resp.status,
'code': getattr(exc, 'code') or _exc_code,
'source': {
'pointer': '/{}'.format(field)
}
}
_errors.append(_error)
else:
_error = {
'id': uuid4(),
'code': getattr(exc, 'code') or _exc_code,
'title': exc.title,
'detail': _('Field value error'),
'status': resp.status
}
_errors.append(_error)
result = ErrorsSchema().dump({'errors': _errors})
resp.text = json.dumps(result, cls=LazyEncoder)
def log_experiment_input(self) -> None:
logger.info(
f'Using the following config: \n{pformat(jsons.dump(self.config))}'
)
dump_to_file(
self.config,
os.path.join(self.path_to_trained_model, CONFIG_FILE_NAME))
if self.comet_experiment:
flat_config = FlatDict(jsons.dump(self.config))
for name, value in flat_config.items():
self.comet_experiment.log_parameter(name, value)
self.comet_experiment.log_metric(MODEL_AVAILABLE_METRIC_NAME,
False)
self.comet_experiment.log_metric(TERMINATED_NORMALLY_METRIC_NAME,
False)
def child_page_recursive(pages,
space_id,
parent_page_id,
table_prefix,
recheck_pages_meet_criteria=False,
config_modified=False):
"""Recursively inserts page information into the database after making requests to the Confluence API.
Args:
pages (dict): A dictionary of pages to crawl through, have a look at the example config for more information.
space_id (int): The top level space_id that the information relates to.
parent_page_id (int): The current pages parent page id.
table_prefix (str): The current database table name prefix.
recheck_pages_meet_criteria (bool): Ensures that all current pages meet the criteria set out in the config file.
If this is False, it will assume that all pages in the database meet the criteria and will only take delta changes for these.
config_modified (bool): Whether the config has been modified since last launch.
"""
# if the child page has not been updated since we last stored the information, then no need to check labels/title!
for page_type in pages:
for page_identifier in pages[page_type]:
# Create tables to store the pages in and the information they contain.
# table = table_prefix + '_' + page_type + '_' + page_identifier
table = table_prefix.replace(
' ', '') + '_' + page_identifier.replace('_', '').replace(
' ', '')[:5].lower()
DatabaseAPI.create_table(table)
info_table = table + '__info'
DatabaseAPI.create_table(info_table, True)
try:
child_pages = ConfluenceAPI.get_child_page_ids(parent_page_id)
except:
logger.warning(
'child_page_recursive: Unable to get child pages for: %s' %
str(parent_page_id))
continue
for child_page_id in child_pages:
# Decision tree to see if the current page meets the criteria provided in the config file.
# if we are not forced to recheck the page meets the criteria then use the pages in the database table.
# else, check to see if the page meets either criteria.
page_meets_criteria = False
if not recheck_pages_meet_criteria and not config_modified:
if DatabaseAPI.check_data_exists(table, parent_page_id,
child_page_id):
# If the page already exists in the database ignore
# checking the page meets the criteria, unless forced to.
page_meets_criteria = True
else:
if page_type == 'titles':
if child_pages[child_page_id][
'name'] == page_identifier:
page_meets_criteria = True
elif page_type == 'labels':
try:
if page_identifier in ConfluenceAPI.get_page_labels(
child_page_id):
# Check that the page meets the criteria given,
# i.e. it is labelled as something/title is something and needs to be updated.
page_meets_criteria = True
except:
logger.warning(
'child_page_recursive: Unable to retrieve labels for: %s'
% str(child_page_id))
continue
if page_meets_criteria:
page_updated = DatabaseAPI.insert_or_update(
table, parent_page_id, child_page_id,
child_pages[child_page_id]['name'],
child_pages[child_page_id]['last_updated'], True)
# If the current page information was updated since the last run,
# delete all children information and re-fill it.
page_content = ''
if page_updated or config_modified:
logger.info(
'Updating information in space %s for page: %s' %
(str(space_id),
child_pages[child_page_id]['name']))
DatabaseAPI.delete(info_table, child_page_id)
try:
page_content = ConfluenceAPI.get_page_content(
child_page_id)
except:
logger.warning(
'child_page_recursive: Unable to get page content for: %s'
% str(child_page_id))
continue
for page_info_type in pages[page_type][page_identifier]:
if page_info_type == 'pages':
child_page_recursive(
pages[page_type][page_identifier]
[page_info_type], space_id, child_page_id,
table, recheck_pages_meet_criteria,
config_modified)
else:
if page_updated or config_modified:
try:
if page_info_type == 'panels':
for panel_identifier in pages[
page_type][page_identifier][
page_info_type]:
panel = FlatDict(
ConfluenceAPI.get_panel(
page_content,
panel_identifier,
space_id))
for k, v in panel.items():
# For each key remove list numbers.
# i.e. FlatDict will put in :0, :1: for each list element.
k = re.sub(':(\d+)', '', k)
k = re.sub(':(\d+):', ':', k)
DatabaseAPI.insert_or_update(
info_table, child_page_id,
k, v,
child_pages[child_page_id]
['last_updated'])
elif page_info_type == 'page_properties':
# Get all page properties and put the values into the database.
page_properties = ConfluenceAPI.get_page_properties(
child_page_id, space_id,
pages[page_type][page_identifier]
[page_info_type])
for page_property in page_properties:
for val in page_properties[
page_property]:
DatabaseAPI.insert_or_update(
info_table, child_page_id,
page_property, val,
child_pages[child_page_id]
['last_updated'])
elif page_info_type == 'headings':
for heading_identifier in pages[
page_type][page_identifier][
page_info_type]:
heading = FlatDict(
ConfluenceAPI.get_heading(
page_content,
heading_identifier))
for k, v in heading.items():
# For each key remove list numbers.
# i.e. FlatDict will put in :0, :1: for each list element.
k = re.sub(':(\d+)', '', k)
k = re.sub(':(\d+):', ':', k)
DatabaseAPI.insert_or_update(
info_table, child_page_id,
k, v,
child_pages[child_page_id]
['last_updated'])
elif page_info_type == 'page':
page_information = FlatDict(
ConfluenceAPI.get_page(
page_content,
child_pages[child_page_id]
['name']))
for k, v in page_information.items():
# For each key remove list numbers.
# i.e. FlatDict will put in :0, :1: for each list element.
k = re.sub(':(\d+)', '', k)
k = re.sub(':(\d+):', ':', k)
DatabaseAPI.insert_or_update(
info_table, child_page_id, k,
v, child_pages[child_page_id]
['last_updated'])
elif page_info_type == 'url':
for url_type in pages[page_type][
page_identifier][
page_info_type]:
url = ConfluenceAPI.get_page_urls(
child_page_id, url_type)
DatabaseAPI.insert_or_update(
info_table, child_page_id,
url_type, url,
child_pages[child_page_id]
['last_updated'])
else:
logger.warning(
'child_page_recursive: Unknown page information retrieval type: %s'
% page_info_type)
except:
logger.error(
'child_page_recursive: Error inserting data for page with id: %s, name: %s'
% (str(child_page_id),
child_pages[child_page_id]['name']))
else:
# Cleanup the ignore, info and default table by removing any information associated with page.
# Child pages get cleaned up by the cleanup method.
DatabaseAPI.delete(table, parent_page_id, child_page_id)
DatabaseAPI.delete(info_table, child_page_id)