def get_page_content(slug):
'''
Get a page content from gh repo (md).
This has a double layer of cache:
- @cache.cached decorator w/ short lived cache for normal operations
- a long terme cache w/o timeout to be able to always render some content
'''
cache_key = f'pages-content-{slug}'
raw_url, gh_url = get_pages_gh_urls(slug)
try:
response = requests.get(raw_url, timeout=5)
# do not cache 404 and forward status code
if response.status_code == 404:
abort(404)
response.raise_for_status()
except requests.exceptions.RequestException as e:
log.exception(f'Error while getting {slug} page from gh: {e}')
content = cache.get(cache_key)
else:
content = response.text
cache.set(cache_key, content)
# no cached version or no content from gh
if not content:
log.error(f'No content found inc. from cache for page {slug}')
abort(503)
return content, gh_url
def objects():
'''
Get a list of schemas from a schema catalog endpoint.
This has a double layer of cache:
- @cache.cached decorator w/ short lived cache for normal operations
- a long terme cache w/o timeout to be able to always render some content
'''
endpoint = current_app.config.get('SCHEMA_CATALOG_URL')
if endpoint is None:
return []
cache_key = 'schema-catalog-objects'
try:
response = requests.get(endpoint, timeout=5)
# do not cache 404 and forward status code
if response.status_code == 404:
raise SchemasCatalogNotFoundException(
f'Schemas catalog does not exist at {endpoint}')
response.raise_for_status()
except requests.exceptions.RequestException as e:
log.exception(
f'Error while getting schema catalog from {endpoint}')
content = cache.get(cache_key)
else:
schemas = response.json().get('schemas', [])
content = [{'id': s['name'], 'label': s['title']} for s in schemas]
cache.set(cache_key, content)
# no cached version or no content
if not content:
log.error(f'No content found inc. from cache for schema catalog')
raise SchemasCacheUnavailableException(
'No content in cache for schema catalog')
return content
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = cache.get(TASKS_LIST_CACHE_KEY) or []
if not tasks:
registered = inspect().registered_tasks()
if registered:
for w, tasks_list in registered.iteritems():
tasks += [t for t in tasks_list if t not in tasks]
cache.set(TASKS_LIST_CACHE_KEY, tasks,
timeout=TASKS_LIST_CACHE_DURATION)
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), task))
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = cache.get(TASKS_LIST_CACHE_KEY) or []
if not tasks:
registered = inspect().registered_tasks()
if registered:
for w, tasks_list in registered.iteritems():
tasks += [t for t in tasks_list if t not in tasks]
cache.set(TASKS_LIST_CACHE_KEY,
tasks,
timeout=TASKS_LIST_CACHE_DURATION)
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), task))
def create_page(app, page, urlset):
key = CACHE_KEY.format(page)
cache.set(key, sitemap.render_page(urlset=urlset))
def create_page(app, page, urlset):
key = CACHE_KEY.format(page)
cache.set(key, sitemap.render_page(urlset=urlset))
def create_page(app, page, urlset):
cache.set(page, sitemap.render_page(urlset=urlset))