def cache(expire, func, *args, **kwargs):
global _cache_engine
_cache_engine = Cache()
# mem args used to build reference id for cache.
mem_args = [object_name(func), ]
# NOTE(cfrademan): This is important, we dont want object address,
# types etc inside of the cache reference. We cannot memoize based
# on args, kwargsargs provided to function containing objects other than
# str, int, float, bytes,
args = list(args) + list(orderdict(kwargs).items())
for arg in args:
if not isinstance(arg, (str, int, float, bytes,)):
mem_args.append(object_name(arg))
else:
raise ValueError("Cache 'callable' not possible with" +
" args/kwargsargs containing values with types" +
" other than 'str', 'int', 'float', 'bytes'")
# create the actual key / reference id.
key = md5sum(pickle.dumps(mem_args))
cached = _cache_engine.load(key)
if cached is not None:
return cached
result = func(*args, **kwargs)
_cache_engine.store(key, result, expire)
return result
def handle_error(self, req, resp, exception, trace):
if isinstance(exception, HTTPError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, AccessDeniedError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, NotFoundError):
log.debug('%s' % (trace))
log.error('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, JSONDecodeError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, FieldError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, ValidationError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
elif isinstance(exception, Error):
log.debug('%s' % (trace))
log.error('%s: %s' % (object_name(exception), exception))
else:
log.debug('%s' % (trace))
log.critical('%s: %s' % (object_name(exception), exception))
def handle_error(self, req, resp, exception, trace):
if isinstance(exception, HTTPError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception),
exception))
title = exception.title
description = exception.description
elif isinstance(exception, AccessDeniedError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception),
exception))
title = "Access Denied"
description = str(exception)
elif isinstance(exception, NotFoundError):
log.debug('%s' % (trace))
log.error('%s: %s' % (object_name(exception),
exception))
title = "Not Found"
description = str(exception)
elif isinstance(exception, JSONDecodeError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception),
exception))
title = "Bad Request (JSON)"
description = str(exception)
elif isinstance(exception, FieldError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception),
exception))
title = "Bad Request (Field)"
description = str(exception)
elif isinstance(exception, ValidationError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception),
exception))
title = "Bad Request"
description = str(exception)
elif isinstance(exception, Error):
log.debug('%s' % (trace))
log.error('%s: %s' % (object_name(exception),
exception))
title = "Error"
description = str(exception)
else:
log.debug('%s' % (trace))
log.critical('%s: %s' % (object_name(exception),
exception))
title = exception.__class__.__name__
description = str(exception)
resp.write(title + " " + description + '\n')
exit(1)
def _memoize(func, *args, **kw):
cache = Cache()
mem_args = args[:num_args]
# frozenset is used to ensure hashability
if kw:
key = object_name(func), mem_args, frozenset(kw.iteritems())
else:
key = object_name(func), mem_args
key = pickle.dumps(key)
cached = cache.load(key)
if cached is not None:
return cached
result = func(*args, **kw)
cache.store(key, result, expiry_time)
return result
def __delitem__(self, key):
try:
del _thread_globals[key]
except KeyError:
try:
del self.__dict__[key]
except KeyError:
raise KeyError("'" + object_name(self) +
"' object has no key '" + key + "'") from None
def __getitem__(self, key):
try:
return _thread_globals[key]
except KeyError:
try:
return self.__dict__[key]
except KeyError:
if key in _context_items:
# Place holder for context - Provides nice error.
return NullError(
NoContextError,
"Working outside of '%s'" % key + " context")
raise KeyError("'" + object_name(self) +
"' object has no key '" + key + "'") from None
def add(self, methods, route, resource, tag=None, cache=0):
"""Add route to view.
The route() method is used to associate a URI template with
a resource. Luxon then maps incoming requests to resources
based on these templates.
URI Template example: "/music/rock"
If the route’s template contains field expressions, any responder that
desires to receive requests for that route must accept arguments named
after the respective field names defined in the template.
A field expression consists of a bracketed field name.
For example, given the following template: "/music/{genre}"
The view would look like:
def genre(self, req, resp, genre):
Args:
methods (list): List of Methods. Use constants in
For example in HTTP...
luxon.constants. can be used ie HTTP_GET
* HTTP_GET
* HTTP_POST
* HTTP_PUT
* HTTP_PATCH
* HTTP_DELETE
route (str): Route resource. (URI Template)
resource (object): Actual view function or method.
Keyword Args:
tag (str): Used to identify rule_set to apply. default: 'None'
"""
methods = to_tuple(methods)
if route[0:6].lower() == "regex:":
route = route[6:]
for method in methods:
method = method.upper()
try:
route = re.compile(route)
self._regex_routes[method].append((
resource,
method,
{},
route,
tag,
cache,
))
self._routes['%s:%s' %
(method, route)] = (resource, method, {},
route, tag, cache)
except KeyError:
self._regex_routes[method] = []
route = re.compile(route)
self._routes['%s:%s' %
(method, route)] = (resource, method, {},
route, tag, cache)
self._regex_routes[method].append(
self._routes['%s:%s' % (method, route)])
except Exception as e:
raise exceptions.Error("Bad RE expression for route '%s'" %
route + ". (%s)" % e)
else:
route = route.strip('/')
for method in methods:
method = method.upper()
try:
if not isinstance(route, retype) and '{' in route:
self._routers[method].add_route(
route, method, resource, tag, cache)
self._routes['%s:%s' %
(method, route)] = (resource, method, {},
route, tag, cache)
except KeyError:
if not isinstance(route, retype) and '{' in route:
self._routers[method] = CompiledRouter()
self._routers[method].add_route(
route, method, resource, tag, cache)
self._routes['%s:%s' %
(method, route)] = (resource, method, {},
route, tag, cache)
self._methods.add(method)
log.info('Added Route: %s' % route + ' Methods: %s' % str(methods) +
' Resource: %s' % object_name(resource) + ' Tag: %s' % tag +
' Cache: %s' % cache)
def __repr__(self):
"""Return representation of thread list.
"""
return repr(object_name(self) + str(self()))
def _log(msg, obj, pool):
log.debug('%s: %s (COUNT: %s, MAX_POOL_SIZE: %s, MAX_OVERFLOW %s' %
(msg, object_name(obj), pool._count,
pool._pool_size, pool._max_overflow))
def github(req, resp):
root_path = g.app.path
mkdir(joinpath(root_path, 'github'))
mkdir(joinpath(root_path, 'docs'))
try:
projects = load(root_path + '/projects.pickle')
except FileNotFoundError:
projects = {}
username = g.app.config.get('github', 'username')
password = g.app.config.get('github', 'password')
tachyonic = GitHub(auth=(username, password))
while True:
try:
teams = {}
github_teams = tachyonic.teams('TachyonicProject')
for github_team in github_teams:
team = github_team['name']
if team == "Author":
continue
teams[team] = {}
github_members = tachyonic.team_members(github_team['id'])
for github_member in github_members:
login = github_member['login']
teams[team][login] = {}
teams[team][login]['github'] = github_member['html_url']
teams[team][login]['avatar'] = github_member['avatar_url']
save(teams, root_path + '/team.pickle', perms=664)
save(tachyonic.projects('TachyonicProject'),
root_path + '/planning.pickle',
perms=664)
found = []
log.info("Getting Repos")
repos = tachyonic.repos('TachyonicProject')
for repo in repos:
name = repo['name']
found.append(name)
description = repo['description']
if name not in projects:
projects[name] = {}
log.info("Scanning Repo " + name)
updated_at = utc(repo['updated_at'])
created_at = utc(repo['created_at'])
pushed_at = utc(repo['pushed_at'])
if (('updated_at' not in projects[name])
or ('updated_at' in projects[name]
and updated_at != projects[name]['updated_at'])
or ('pushed_at' not in projects[name])
or ('pushed_at' in projects[name]
and pushed_at != projects[name]['pushed_at'])):
projects[name]['created_at'] = created_at
projects[name]['description'] = description
projects[name]['clone_url'] = repo['clone_url']
log.info("Getting Branches for %s" % name)
branches = tachyonic.branches('TachyonicProject', name)
branches = [branch['name'] for branch in branches]
projects[name]['branches'] = branches
log.info("Getting Tags for %s" % name)
tags = tachyonic.tags('TachyonicProject', name)
tags = [tag['name'] for tag in tags]
projects[name]['tags'] = tags
projects[name]['refs'] = version_order(branches + tags)
projects[name]['doc_refs'] = {}
else:
log.info("Project %s Already up-to-date (%s)" % (
name,
updated_at,
))
projects[name]['updated_at'] = updated_at
projects[name]['pushed_at'] = pushed_at
if 'updated_doc' not in projects[name]:
projects[name]['updated_doc'] = {}
for ref in projects[name]['refs']:
current_datetime = now()
if ref in projects[name]['updated_doc']:
commits = tachyonic.commits(
'TachyonicProject',
name,
sha=ref,
since=format_iso8601(
projects[name]['updated_doc'][ref]))
if len(commits) == 0:
log.info("Documentation" + " '%s/%s'" % (
name,
ref,
) + " Already up-to-date (%s)" % updated_at)
continue
venv_dir = "%s/github/%s_%s" % (
root_path,
name,
ref,
)
doc_dir = "%s/docs/%s_%s" % (
root_path,
name,
ref,
)
src_path = venv_dir + '/' + name
log.info("Creating Virtual Environment '%s'" % venv_dir)
create_env(str(venv_dir), wipe=True, site_packages=False)
clone(projects[name]['clone_url'], src_path)
if (exists(src_path + '/docs/source/conf.py')
and exists(src_path + '/docs/Makefile')):
log.info("Bulding '%s/%s'" % (
name,
ref,
))
projects[name]['doc_refs'][ref] = True
info = build_doc(root_path, venv_dir, src_path, ref,
doc_dir, name)
updated(name, ref, info)
else:
projects[name]['doc_refs'][ref] = False
log.warning("No Sphinx docs found '%s/%s'" % (
name,
ref,
))
projects[name]['updated_doc'][ref] = current_datetime
save(projects, root_path + '/projects.pickle', perms=664)
events = []
events_ordered = []
git_events = tachyonic.events('TachyonicProject')
for pj in projects.copy():
if pj not in found:
del projects[pj]
else:
for event in git_events:
type = event['type']
payload = event['payload']
if type == 'PullRequestEvent':
pr = payload['pull_request']
merged = pr['merged']
base = pr['base']
ref = base['ref']
if merged is True:
merged_at = utc(pr['merged_at'])
events.append((merged_at, "Code Updated",
"Repo " + pj + "/" + ref + ""))
for item in sorted(events, key=operator.itemgetter(0)):
events_ordered.append(item)
events_ordered = list(reversed(events_ordered))
save(events_ordered[0:10], root_path + '/events.pickle', perms=664)
save(projects, root_path + '/projects.pickle', perms=664)
log.info('Infinite loop sleeping 5 Minutes')
sleep(300)
except KeyboardInterrupt:
print("Control-C closed / Killed")
break
except ExecuteError as e:
handle_error(e.title, e.description)
except Exception as e:
trace = str(traceback.format_exc())
error = '%s: %s' % (object_name(e), e)
handle_error(error, trace)
def __str__(self):
"""Return string of dictionary.
"""
return str(object_name(self) + str(self()))
def __call__(self, *args, **kwargs):
"""Application Request Interface.
A clean request and response object is provided to the interface that
is unique this to this thread.
It passes any args and kwargs to the interface.
Response object is returned.
"""
try:
with Timer() as elapsed:
# Request Object.
request = g.current_request = self._REQUEST_CLASS(
*args, **kwargs)
# Response Object.
response = self._RESPONSE_CLASS(*args, **kwargs)
# Set Response object for request.
request.response = response
# Process the middleware 'pre' method before routing it
for middleware in g.middleware_pre:
middleware(request, response)
# Route Object.
resource, method, r_kwargs, target, tag = g.router.find( \
request.method, \
request.route)
# Set route tag in requests.
request.tag = tag
# Execute Routed View.
try:
# Run View method.
if resource is not None:
# Process the middleware 'resource' after routing it
for middleware in g.middleware_resource:
middleware(request, response)
view = resource(request, response, **r_kwargs)
if view is not None:
response.body(view)
else:
raise NotFound('Route not found' +
" Method '%s'" % request.method +
" Route '%s'" % request.route)
finally:
# Process the middleware 'post' at the end
for middleware in g.middleware_post:
middleware(request, response)
# Return response object.
return response
except HTTPError as exception:
trace = str(traceback.format_exc())
if log.debug_mode():
log.debug('%s' % (trace))
else:
log.info('%s: %s' % (object_name(exception), exception))
self._proxy_handle_error(request, response, exception, trace)
# Return response object.
return response
except Error as exception:
trace = str(traceback.format_exc())
if log.debug_mode():
log.debug('%s' % (trace))
else:
log.error('%s: %s' % (object_name(exception), exception))
self._proxy_handle_error(request, response, exception, trace)
# Return response object.
return response
except Exception as exception:
trace = str(traceback.format_exc())
if log.debug_mode():
log.debug('%s' % (trace))
else:
log.error('%s: %s' % (object_name(exception), exception))
self._proxy_handle_error(request, response, exception, trace)
# Return response object.
return response
finally:
# Completed Request
log.info('Completed Request', timer=elapsed())
def __repr__(self):
"""Return representation of dictionary.
"""
return repr(object_name(self) + str(self()))
def __str__(self):
"""Return string of thread list.
"""
return str(object_name(self) + str(self()))
def __delattr__(self, attr):
try:
del self[attr]
except KeyError:
raise AttributeError("'" + object_name(self) +
"' object has no '" + attr + "'") from None
def daemon(root_path, config):
try:
projects = load(root_path + '/projects.pickle')
except FileNotFoundError:
projects = {}
email = config.get('github', 'email')
rcpt = config.get('github', 'rcpt')
username = config.get('github', 'username')
password = config.get('github', 'password')
tachyonic = GitHub('https://api.github.com', auth=(username, password))
while True:
try:
save(tachyonic.projects('TachyonicProject'),
root_path + '/planning.pickle')
found = []
log.info("Getting Repos")
repos = tachyonic.repos('TachyonicProject')
for repo in repos:
name = repo['name']
found.append(name)
description = repo['description']
if name not in projects:
projects[name] = {}
log.info("Scanning Repo " + name)
updated_at = repo['updated_at']
created_at = repo['updated_at']
projects[name]['description'] = description
projects[name]['created_at'] = created_at
projects[name]['events'] = tachyonic.events(
'TachyonicProject', name)
if (name in projects and 'updated_at' in projects[name]
and updated_at == projects[name]['updated_at']):
log.info("Already up-to-date (%s)" % updated_at)
continue
projects[name]['updated_at'] = updated_at
clone_url = repo['clone_url']
projects[name]['clone_url'] = clone_url
log.info("Getting Branches for %s" % name)
branches = tachyonic.branches('TachyonicProject', name)
log.info("Getting Tags for %s" % name)
tags = tachyonic.tags('TachyonicProject', name)
refs = branches + tags
projects[name]['refs'] = refs
projects[name]['branches'] = branches
projects[name]['tags'] = tags
projects[name]['doc_refs'] = {}
for ref in refs:
venv_dir = "%s/github/%s_%s" % (
root_path,
name,
ref,
)
doc_dir = "%s/docs/%s_%s" % (
root_path,
name,
ref,
)
src_path = venv_dir + '/' + name
log.info("Creating Virtual Environment '%s'" % venv_dir)
create_env(venv_dir, wipe=True, site_packages=True)
clone(clone_url, src_path)
if (os.path.exists(src_path + '/docs/source/conf.py')
and os.path.exists(src_path + '/docs/Makefile')):
log.info("Bulding '%s' ref '%s'" % (
name,
ref,
))
projects[name]['doc_refs'][ref] = True
build_doc(root_path, venv_dir, src_path, ref, doc_dir,
name)
else:
projects[name]['doc_refs'][ref] = False
log.warning("No Sphinx docs found ref '%s'" % ref)
save(projects, root_path + '/projects.pickle')
for pj in projects:
if pj not in found:
del projects[pj]
save(projects, root_path + '/projects.pickle')
log.info('Infinite loop sleeping 121 seconds')
sleep(121)
except KeyboardInterrupt:
print("Control-C closed / Killed")
break
except Exception as e:
trace = str(traceback.format_exc())
error = '%s: %s' % (object_name(e), e)
try:
if e.status == 401:
if log.debug_mode():
log.debug(trace)
else:
log.error(error)
send_email(email,
rcpt,
subject='GitHub TachWeb Error %s' % error,
body=trace)
break
log.error(e)
except AttributeError:
if log.debug_mode():
log.debug(trace)
else:
log.error(error)
send_email(email,
rcpt,
subject='GitHub TachWeb Error %s' % error,
body=trace)
log.info('Error in loop sleeping 5 minutes')
sleep(300)
def handle_error(self, req, resp, exception, trace):
# Parse Exceptions.
resp.cache_control = "no-store, no-cache, max-age=0"
if isinstance(exception, HTTPError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
resp.status = exception.status
title = exception.title
description = exception.description
for header in exception.headers:
resp.set_header(header, exception.headers[header])
elif isinstance(exception, AccessDeniedError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Access Denied"
description = str(exception)
if isinstance(exception, TokenExpiredError):
resp.set_header('X-Expired-Token', 'true')
resp.status = 403
elif isinstance(exception, NotFoundError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Not Found"
description = str(exception)
resp.status = 404
elif isinstance(exception, JSONDecodeError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Bad Request (JSON)"
description = str(exception)
resp.status = 400
elif isinstance(exception, FieldError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Bad Request (Field)"
description = str(exception)
resp.status = 400
elif isinstance(exception, ValidationError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Bad Request"
description = str(exception)
resp.status = 400
elif isinstance(exception, Error):
log.debug('%s' % (trace))
log.error('%s: %s' % (object_name(exception), exception))
title = "Error"
description = str(exception)
resp.status = 500
elif isinstance(exception, ValueError):
log.debug('%s' % (trace))
log.warning('%s: %s' % (object_name(exception), exception))
title = "Bad Request"
description = str(exception)
resp.status = 400
else:
log.debug('%s' % (trace))
log.critical('%s: %s' % (object_name(exception), exception))
title = exception.__class__.__name__
description = str(exception)
resp.status = 500
# Generate Error Response
if req.is_ajax and register._ajax_error_template:
# if AJAX Template and AJAX Request.
resp.content_type = TEXT_HTML
try:
resp.body(
render_template(register._ajax_error_template,
error_title=title,
error_description=description))
except Exception:
trace = str(traceback.format_exc())
log.error('Unable to render ajax error template\n%s' % trace)
resp.body(error_ajax(title, description))
elif register._error_template:
# If Error Template.
resp.content_type = TEXT_HTML
try:
resp.body(
render_template(register._error_template,
error_title=title,
error_description=description))
except Exception:
trace = str(traceback.format_exc())
log.error('Unable to render page error template\n%s' % trace)
resp.body(error_page(title, description))
elif resp.content_type is None or 'json' in resp.content_type.lower():
# Check if JSON Content and provide JSON Error.
to_return = {}
to_return['error'] = {}
to_return['error']['title'] = title
to_return['error']['description'] = description
resp.body(to_return)
elif 'html' in resp.content_type.lower():
# Else if HTML Content Respond with Generic HTML Error
resp.content_type = TEXT_HTML
resp.body(error_page(title, description))
else:
# Else Generic TEXT Error.
resp.content_type = TEXT_PLAIN
resp.body(title + ' ' + description)
