def cache(expire, func, *args, **kwargs): global _cache_engine _cache_engine = Cache() # mem args used to build reference id for cache. mem_args = [object_name(func), ] # NOTE(cfrademan): This is important, we dont want object address, # types etc inside of the cache reference. We cannot memoize based # on args, kwargsargs provided to function containing objects other than # str, int, float, bytes, args = list(args) + list(orderdict(kwargs).items()) for arg in args: if not isinstance(arg, (str, int, float, bytes,)): mem_args.append(object_name(arg)) else: raise ValueError("Cache 'callable' not possible with" + " args/kwargsargs containing values with types" + " other than 'str', 'int', 'float', 'bytes'") # create the actual key / reference id. key = md5sum(pickle.dumps(mem_args)) cached = _cache_engine.load(key) if cached is not None: return cached result = func(*args, **kwargs) _cache_engine.store(key, result, expire) return result
def handle_error(self, req, resp, exception, trace): if isinstance(exception, HTTPError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, AccessDeniedError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, NotFoundError): log.debug('%s' % (trace)) log.error('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, JSONDecodeError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, FieldError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, ValidationError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) elif isinstance(exception, Error): log.debug('%s' % (trace)) log.error('%s: %s' % (object_name(exception), exception)) else: log.debug('%s' % (trace)) log.critical('%s: %s' % (object_name(exception), exception))
def handle_error(self, req, resp, exception, trace): if isinstance(exception, HTTPError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = exception.title description = exception.description elif isinstance(exception, AccessDeniedError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Access Denied" description = str(exception) elif isinstance(exception, NotFoundError): log.debug('%s' % (trace)) log.error('%s: %s' % (object_name(exception), exception)) title = "Not Found" description = str(exception) elif isinstance(exception, JSONDecodeError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request (JSON)" description = str(exception) elif isinstance(exception, FieldError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request (Field)" description = str(exception) elif isinstance(exception, ValidationError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request" description = str(exception) elif isinstance(exception, Error): log.debug('%s' % (trace)) log.error('%s: %s' % (object_name(exception), exception)) title = "Error" description = str(exception) else: log.debug('%s' % (trace)) log.critical('%s: %s' % (object_name(exception), exception)) title = exception.__class__.__name__ description = str(exception) resp.write(title + " " + description + '\n') exit(1)
def _memoize(func, *args, **kw): cache = Cache() mem_args = args[:num_args] # frozenset is used to ensure hashability if kw: key = object_name(func), mem_args, frozenset(kw.iteritems()) else: key = object_name(func), mem_args key = pickle.dumps(key) cached = cache.load(key) if cached is not None: return cached result = func(*args, **kw) cache.store(key, result, expiry_time) return result
def __delitem__(self, key): try: del _thread_globals[key] except KeyError: try: del self.__dict__[key] except KeyError: raise KeyError("'" + object_name(self) + "' object has no key '" + key + "'") from None
def __getitem__(self, key): try: return _thread_globals[key] except KeyError: try: return self.__dict__[key] except KeyError: if key in _context_items: # Place holder for context - Provides nice error. return NullError( NoContextError, "Working outside of '%s'" % key + " context") raise KeyError("'" + object_name(self) + "' object has no key '" + key + "'") from None
def add(self, methods, route, resource, tag=None, cache=0): """Add route to view. The route() method is used to associate a URI template with a resource. Luxon then maps incoming requests to resources based on these templates. URI Template example: "/music/rock" If the route’s template contains field expressions, any responder that desires to receive requests for that route must accept arguments named after the respective field names defined in the template. A field expression consists of a bracketed field name. For example, given the following template: "/music/{genre}" The view would look like: def genre(self, req, resp, genre): Args: methods (list): List of Methods. Use constants in For example in HTTP... luxon.constants. can be used ie HTTP_GET * HTTP_GET * HTTP_POST * HTTP_PUT * HTTP_PATCH * HTTP_DELETE route (str): Route resource. (URI Template) resource (object): Actual view function or method. Keyword Args: tag (str): Used to identify rule_set to apply. default: 'None' """ methods = to_tuple(methods) if route[0:6].lower() == "regex:": route = route[6:] for method in methods: method = method.upper() try: route = re.compile(route) self._regex_routes[method].append(( resource, method, {}, route, tag, cache, )) self._routes['%s:%s' % (method, route)] = (resource, method, {}, route, tag, cache) except KeyError: self._regex_routes[method] = [] route = re.compile(route) self._routes['%s:%s' % (method, route)] = (resource, method, {}, route, tag, cache) self._regex_routes[method].append( self._routes['%s:%s' % (method, route)]) except Exception as e: raise exceptions.Error("Bad RE expression for route '%s'" % route + ". (%s)" % e) else: route = route.strip('/') for method in methods: method = method.upper() try: if not isinstance(route, retype) and '{' in route: self._routers[method].add_route( route, method, resource, tag, cache) self._routes['%s:%s' % (method, route)] = (resource, method, {}, route, tag, cache) except KeyError: if not isinstance(route, retype) and '{' in route: self._routers[method] = CompiledRouter() self._routers[method].add_route( route, method, resource, tag, cache) self._routes['%s:%s' % (method, route)] = (resource, method, {}, route, tag, cache) self._methods.add(method) log.info('Added Route: %s' % route + ' Methods: %s' % str(methods) + ' Resource: %s' % object_name(resource) + ' Tag: %s' % tag + ' Cache: %s' % cache)
def __repr__(self): """Return representation of thread list. """ return repr(object_name(self) + str(self()))
def _log(msg, obj, pool): log.debug('%s: %s (COUNT: %s, MAX_POOL_SIZE: %s, MAX_OVERFLOW %s' % (msg, object_name(obj), pool._count, pool._pool_size, pool._max_overflow))
def github(req, resp): root_path = g.app.path mkdir(joinpath(root_path, 'github')) mkdir(joinpath(root_path, 'docs')) try: projects = load(root_path + '/projects.pickle') except FileNotFoundError: projects = {} username = g.app.config.get('github', 'username') password = g.app.config.get('github', 'password') tachyonic = GitHub(auth=(username, password)) while True: try: teams = {} github_teams = tachyonic.teams('TachyonicProject') for github_team in github_teams: team = github_team['name'] if team == "Author": continue teams[team] = {} github_members = tachyonic.team_members(github_team['id']) for github_member in github_members: login = github_member['login'] teams[team][login] = {} teams[team][login]['github'] = github_member['html_url'] teams[team][login]['avatar'] = github_member['avatar_url'] save(teams, root_path + '/team.pickle', perms=664) save(tachyonic.projects('TachyonicProject'), root_path + '/planning.pickle', perms=664) found = [] log.info("Getting Repos") repos = tachyonic.repos('TachyonicProject') for repo in repos: name = repo['name'] found.append(name) description = repo['description'] if name not in projects: projects[name] = {} log.info("Scanning Repo " + name) updated_at = utc(repo['updated_at']) created_at = utc(repo['created_at']) pushed_at = utc(repo['pushed_at']) if (('updated_at' not in projects[name]) or ('updated_at' in projects[name] and updated_at != projects[name]['updated_at']) or ('pushed_at' not in projects[name]) or ('pushed_at' in projects[name] and pushed_at != projects[name]['pushed_at'])): projects[name]['created_at'] = created_at projects[name]['description'] = description projects[name]['clone_url'] = repo['clone_url'] log.info("Getting Branches for %s" % name) branches = tachyonic.branches('TachyonicProject', name) branches = [branch['name'] for branch in branches] projects[name]['branches'] = branches log.info("Getting Tags for %s" % name) tags = tachyonic.tags('TachyonicProject', name) tags = [tag['name'] for tag in tags] projects[name]['tags'] = tags projects[name]['refs'] = version_order(branches + tags) projects[name]['doc_refs'] = {} else: log.info("Project %s Already up-to-date (%s)" % ( name, updated_at, )) projects[name]['updated_at'] = updated_at projects[name]['pushed_at'] = pushed_at if 'updated_doc' not in projects[name]: projects[name]['updated_doc'] = {} for ref in projects[name]['refs']: current_datetime = now() if ref in projects[name]['updated_doc']: commits = tachyonic.commits( 'TachyonicProject', name, sha=ref, since=format_iso8601( projects[name]['updated_doc'][ref])) if len(commits) == 0: log.info("Documentation" + " '%s/%s'" % ( name, ref, ) + " Already up-to-date (%s)" % updated_at) continue venv_dir = "%s/github/%s_%s" % ( root_path, name, ref, ) doc_dir = "%s/docs/%s_%s" % ( root_path, name, ref, ) src_path = venv_dir + '/' + name log.info("Creating Virtual Environment '%s'" % venv_dir) create_env(str(venv_dir), wipe=True, site_packages=False) clone(projects[name]['clone_url'], src_path) if (exists(src_path + '/docs/source/conf.py') and exists(src_path + '/docs/Makefile')): log.info("Bulding '%s/%s'" % ( name, ref, )) projects[name]['doc_refs'][ref] = True info = build_doc(root_path, venv_dir, src_path, ref, doc_dir, name) updated(name, ref, info) else: projects[name]['doc_refs'][ref] = False log.warning("No Sphinx docs found '%s/%s'" % ( name, ref, )) projects[name]['updated_doc'][ref] = current_datetime save(projects, root_path + '/projects.pickle', perms=664) events = [] events_ordered = [] git_events = tachyonic.events('TachyonicProject') for pj in projects.copy(): if pj not in found: del projects[pj] else: for event in git_events: type = event['type'] payload = event['payload'] if type == 'PullRequestEvent': pr = payload['pull_request'] merged = pr['merged'] base = pr['base'] ref = base['ref'] if merged is True: merged_at = utc(pr['merged_at']) events.append((merged_at, "Code Updated", "Repo " + pj + "/" + ref + "")) for item in sorted(events, key=operator.itemgetter(0)): events_ordered.append(item) events_ordered = list(reversed(events_ordered)) save(events_ordered[0:10], root_path + '/events.pickle', perms=664) save(projects, root_path + '/projects.pickle', perms=664) log.info('Infinite loop sleeping 5 Minutes') sleep(300) except KeyboardInterrupt: print("Control-C closed / Killed") break except ExecuteError as e: handle_error(e.title, e.description) except Exception as e: trace = str(traceback.format_exc()) error = '%s: %s' % (object_name(e), e) handle_error(error, trace)
def __str__(self): """Return string of dictionary. """ return str(object_name(self) + str(self()))
def __call__(self, *args, **kwargs): """Application Request Interface. A clean request and response object is provided to the interface that is unique this to this thread. It passes any args and kwargs to the interface. Response object is returned. """ try: with Timer() as elapsed: # Request Object. request = g.current_request = self._REQUEST_CLASS( *args, **kwargs) # Response Object. response = self._RESPONSE_CLASS(*args, **kwargs) # Set Response object for request. request.response = response # Process the middleware 'pre' method before routing it for middleware in g.middleware_pre: middleware(request, response) # Route Object. resource, method, r_kwargs, target, tag = g.router.find( \ request.method, \ request.route) # Set route tag in requests. request.tag = tag # Execute Routed View. try: # Run View method. if resource is not None: # Process the middleware 'resource' after routing it for middleware in g.middleware_resource: middleware(request, response) view = resource(request, response, **r_kwargs) if view is not None: response.body(view) else: raise NotFound('Route not found' + " Method '%s'" % request.method + " Route '%s'" % request.route) finally: # Process the middleware 'post' at the end for middleware in g.middleware_post: middleware(request, response) # Return response object. return response except HTTPError as exception: trace = str(traceback.format_exc()) if log.debug_mode(): log.debug('%s' % (trace)) else: log.info('%s: %s' % (object_name(exception), exception)) self._proxy_handle_error(request, response, exception, trace) # Return response object. return response except Error as exception: trace = str(traceback.format_exc()) if log.debug_mode(): log.debug('%s' % (trace)) else: log.error('%s: %s' % (object_name(exception), exception)) self._proxy_handle_error(request, response, exception, trace) # Return response object. return response except Exception as exception: trace = str(traceback.format_exc()) if log.debug_mode(): log.debug('%s' % (trace)) else: log.error('%s: %s' % (object_name(exception), exception)) self._proxy_handle_error(request, response, exception, trace) # Return response object. return response finally: # Completed Request log.info('Completed Request', timer=elapsed())
def __repr__(self): """Return representation of dictionary. """ return repr(object_name(self) + str(self()))
def __str__(self): """Return string of thread list. """ return str(object_name(self) + str(self()))
def __delattr__(self, attr): try: del self[attr] except KeyError: raise AttributeError("'" + object_name(self) + "' object has no '" + attr + "'") from None
def daemon(root_path, config): try: projects = load(root_path + '/projects.pickle') except FileNotFoundError: projects = {} email = config.get('github', 'email') rcpt = config.get('github', 'rcpt') username = config.get('github', 'username') password = config.get('github', 'password') tachyonic = GitHub('https://api.github.com', auth=(username, password)) while True: try: save(tachyonic.projects('TachyonicProject'), root_path + '/planning.pickle') found = [] log.info("Getting Repos") repos = tachyonic.repos('TachyonicProject') for repo in repos: name = repo['name'] found.append(name) description = repo['description'] if name not in projects: projects[name] = {} log.info("Scanning Repo " + name) updated_at = repo['updated_at'] created_at = repo['updated_at'] projects[name]['description'] = description projects[name]['created_at'] = created_at projects[name]['events'] = tachyonic.events( 'TachyonicProject', name) if (name in projects and 'updated_at' in projects[name] and updated_at == projects[name]['updated_at']): log.info("Already up-to-date (%s)" % updated_at) continue projects[name]['updated_at'] = updated_at clone_url = repo['clone_url'] projects[name]['clone_url'] = clone_url log.info("Getting Branches for %s" % name) branches = tachyonic.branches('TachyonicProject', name) log.info("Getting Tags for %s" % name) tags = tachyonic.tags('TachyonicProject', name) refs = branches + tags projects[name]['refs'] = refs projects[name]['branches'] = branches projects[name]['tags'] = tags projects[name]['doc_refs'] = {} for ref in refs: venv_dir = "%s/github/%s_%s" % ( root_path, name, ref, ) doc_dir = "%s/docs/%s_%s" % ( root_path, name, ref, ) src_path = venv_dir + '/' + name log.info("Creating Virtual Environment '%s'" % venv_dir) create_env(venv_dir, wipe=True, site_packages=True) clone(clone_url, src_path) if (os.path.exists(src_path + '/docs/source/conf.py') and os.path.exists(src_path + '/docs/Makefile')): log.info("Bulding '%s' ref '%s'" % ( name, ref, )) projects[name]['doc_refs'][ref] = True build_doc(root_path, venv_dir, src_path, ref, doc_dir, name) else: projects[name]['doc_refs'][ref] = False log.warning("No Sphinx docs found ref '%s'" % ref) save(projects, root_path + '/projects.pickle') for pj in projects: if pj not in found: del projects[pj] save(projects, root_path + '/projects.pickle') log.info('Infinite loop sleeping 121 seconds') sleep(121) except KeyboardInterrupt: print("Control-C closed / Killed") break except Exception as e: trace = str(traceback.format_exc()) error = '%s: %s' % (object_name(e), e) try: if e.status == 401: if log.debug_mode(): log.debug(trace) else: log.error(error) send_email(email, rcpt, subject='GitHub TachWeb Error %s' % error, body=trace) break log.error(e) except AttributeError: if log.debug_mode(): log.debug(trace) else: log.error(error) send_email(email, rcpt, subject='GitHub TachWeb Error %s' % error, body=trace) log.info('Error in loop sleeping 5 minutes') sleep(300)
def handle_error(self, req, resp, exception, trace): # Parse Exceptions. resp.cache_control = "no-store, no-cache, max-age=0" if isinstance(exception, HTTPError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) resp.status = exception.status title = exception.title description = exception.description for header in exception.headers: resp.set_header(header, exception.headers[header]) elif isinstance(exception, AccessDeniedError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Access Denied" description = str(exception) if isinstance(exception, TokenExpiredError): resp.set_header('X-Expired-Token', 'true') resp.status = 403 elif isinstance(exception, NotFoundError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Not Found" description = str(exception) resp.status = 404 elif isinstance(exception, JSONDecodeError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request (JSON)" description = str(exception) resp.status = 400 elif isinstance(exception, FieldError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request (Field)" description = str(exception) resp.status = 400 elif isinstance(exception, ValidationError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request" description = str(exception) resp.status = 400 elif isinstance(exception, Error): log.debug('%s' % (trace)) log.error('%s: %s' % (object_name(exception), exception)) title = "Error" description = str(exception) resp.status = 500 elif isinstance(exception, ValueError): log.debug('%s' % (trace)) log.warning('%s: %s' % (object_name(exception), exception)) title = "Bad Request" description = str(exception) resp.status = 400 else: log.debug('%s' % (trace)) log.critical('%s: %s' % (object_name(exception), exception)) title = exception.__class__.__name__ description = str(exception) resp.status = 500 # Generate Error Response if req.is_ajax and register._ajax_error_template: # if AJAX Template and AJAX Request. resp.content_type = TEXT_HTML try: resp.body( render_template(register._ajax_error_template, error_title=title, error_description=description)) except Exception: trace = str(traceback.format_exc()) log.error('Unable to render ajax error template\n%s' % trace) resp.body(error_ajax(title, description)) elif register._error_template: # If Error Template. resp.content_type = TEXT_HTML try: resp.body( render_template(register._error_template, error_title=title, error_description=description)) except Exception: trace = str(traceback.format_exc()) log.error('Unable to render page error template\n%s' % trace) resp.body(error_page(title, description)) elif resp.content_type is None or 'json' in resp.content_type.lower(): # Check if JSON Content and provide JSON Error. to_return = {} to_return['error'] = {} to_return['error']['title'] = title to_return['error']['description'] = description resp.body(to_return) elif 'html' in resp.content_type.lower(): # Else if HTML Content Respond with Generic HTML Error resp.content_type = TEXT_HTML resp.body(error_page(title, description)) else: # Else Generic TEXT Error. resp.content_type = TEXT_PLAIN resp.body(title + ' ' + description)