def parse_view_result(self, rows, tag): # here we are given multiple float values for the same document, # we parse them as decimals and sum # ['loan_amount', 1002.2], {'_id': 'merchant_record_1'} # ['loan_amount', 2001.2], {'_id': 'merchant_record_2'} # ... # is resolved to the following: # [('merchant_record_1', decimal.Decimal('1002.20')), # ('merchant_record_2', decimal.Decimal('2001.20'))] indexes = dict() # doc_id -> index in result table result = list() for row in rows: if row[1] and self.id_key in row[1]: doc_id = row[1][self.id_key] else: doc_id = row[2] if doc_id not in indexes: indexes[doc_id] = len(result) result.append((doc_id, decimal.Decimal())) try: v = decimal.Decimal(str(row[0][1])) except: error.handle_exception(None, None, "Failed parsing decimal value") continue else: result[indexes[doc_id]] = (doc_id, result[indexes[doc_id]][1] + v) return ParsedIndex(result, keep_value=True)
def load(module_name, name): log.info('application', "Importing application %s from module %s", name, module_name) module = sys.modules.get(module_name) if module: log.warning('application', "Application module %s has already been loaded. ", module_name) else: module = reflect.named_module(module_name) application = getattr(module, name, None) if application is None: raise ValueError('Module %s has no attribute %s' % (module_name, name)) if not IApplication.providedBy(application): raise ValueError('Variable %s.%s should provide IApplication interface' % (module_name, name)) try: application.load() except Exception as e: error.handle_exception( 'application', e, 'Error loading application: %s', application.name) application.unload() raise else: get_application_registry().register(application) log.info('application', "Loading application %s complete.", name)
def migration_script(connection): log.info("script", "Running the migration script.") for application in applications.get_application_registry().itervalues(): keys = ApplicationVersions.key_for(application.name) version_doc = yield connection.query_view(ApplicationVersions, **keys) if not version_doc: to_run = application.get_migrations() version_doc = ApplicationVersion(name=unicode(application.name)) else: version_doc = version_doc[0] to_run = [(version, migration) for version, migration in application.get_migrations() if version > version_doc.version] if not to_run: log.info("script", "There are no migrations for application %s " "from version %s to %s", application.name, version_doc.version, application.version) continue try: for version, migration in to_run: yield migration.run(connection._database) if isinstance(version, str): version = unicode(version) version_doc.version = version yield connection.save_document(version_doc) log.info("script", "Successfully applied migration %r", migration) except Exception as e: error.handle_exception("script", e, "Failed applying migration %r", migration) continue
def view_aterator(connection, callback, view, view_keys=dict(), args=tuple(), kwargs=dict(), per_page=15, consume_errors=True): ''' Asynchronous iterator for the view. Downloads a view in pages and calls the callback for each row. This helps avoid transfering data in huge datachunks. ''' skip = 0 while True: keys = dict(view_keys) keys.update(dict(skip=skip, limit=per_page)) records = yield connection.query_view(view, **keys) log.debug('view_aterator', "Fetched %d records of the view: %s", len(records), view.name) skip += len(records) for record in records: try: yield callback(connection, record, *args, **kwargs) except Exception as e: error.handle_exception( 'view_aterator', e, "Callback %s failed its iteration on a row %r", callback.__name__, record) if not consume_errors: raise e if not records: break
def render_select_response(value, context, *args, **kwargs): cls = type(context['model']) if not cls._query_set_factory: # query set collection is created only once per class type factory = MetaQueryResult.new(cls) factory.annotate_meta('json', 'render-as-list') cls._query_set_factory = factory if cls._fetch_documents_set: context['result'].update(value) # convert all the aggregate values using their IValueInfo if kwargs.get('aggregate'): raw_values = context['result'].aggregations context['result'].aggregations = dict() for index, name in enumerate(kwargs['aggregate']): value_info = cls._model_aggregations[name][0] v = raw_values[index] try: published = value_info.publish(v) except Exception as e: error.handle_exception( None, e, "Failed publishing the result %r", v) else: context['result'].aggregations[name] = published result = cls._query_set_factory(context['source'], context['result']) return result.initiate(view=context['view'], officer=context.get('officer'), aspect=context.get('aspect'))
def _process_request(self, priv_request): self.log( "%s on %s from %s:%s", priv_request.method, priv_request.uri, priv_request.client.host, priv_request.client.port, ) request = None response = None try: # First create request and response parsing HTTP headers and so request = Request(self, priv_request) response = Response(self, request) except Exception, e: # Early error, just log and respond something that make sense msg = "Exception during HTTP request creation" error.handle_exception(self, e, msg) priv_request.setResponseCode(http.Status.INTERNAL_SERVER_ERROR) priv_request.setHeader("content-type", "text/plain") return "Error: %s" % e
def startup(self, port): try: self._tcp_listener = reactor.listenTCP(port, self._dns_fact) except error.CannotListenError, e: feat_error.handle_exception( self, e, "Error listening on TCP on port %r", port) return False
def _initiate_tunneling(self, tconfig): assert isinstance(tconfig, config.TunnelConfig), str(type(tconfig)) try: host = tconfig.host port = int(tconfig.port) p12 = tconfig.p12 port_range = range(port, port + TUNNELING_PORT_COUNT) self.info("Setting up tunneling on %s ports %d-%d " "using PKCS12 %r", host, port_range[0], port_range[-1], p12) csec = security.ClientContextFactory(p12_filename=p12, verify_ca_from_p12=True) cpol = security.ClientPolicy(csec) ssec = security.ServerContextFactory(p12_filename=p12, verify_ca_from_p12=True, enforce_cert=True) spol = security.ServerPolicy(ssec) backend = tunneling.Backend(host, port_range, client_security_policy=cpol, server_security_policy=spol) frontend = tunneling.Tunneling(backend) return frontend except Exception as e: msg = "Failed to setup tunneling backend" error.handle_exception(self, e, msg) return None
def _side_effect_wrapper(callable, args, kwargs, name): section_state = fiber.get_state() if section_state is not None: # We are in a woven section entry = section_state.get(JOURNAL_ENTRY_TAG, None) if entry is not None: # We are in a replayable section mode = section_state.get(RECMODE_TAG, None) if mode == JournalMode.replay: return entry.next_side_effect(name, *args, **kwargs) # Create a side-effect entry effect = entry.new_side_effect(name, *args, **kwargs) # Keep it in the replayable section state section_state[SIDE_EFFECT_TAG] = effect # Break the fiber to allow new replayable sections fiber.break_fiber() # Keep the side-effect entry to detect we are in one fiber.set_stack_var(SIDE_EFFECT_TAG, effect) try: result = callable(*args, **kwargs) result = _check_side_effet_result(result, name) effect.set_result(result) effect.commit() return result except Exception, e: #FIXME: handle exceptions in side effects properly error.handle_exception(None, e, "Exception raised by side-effect %s", reflect.canonical_name(callable)) raise
def _create_gateway(self, gconfig): assert isinstance(gconfig, config.GatewayConfig), str(type(gconfig)) try: port = int(gconfig.port) p12 = gconfig.p12 allow_tcp = gconfig.allow_tcp range = (port, port + GATEWAY_PORT_COUNT) if not os.path.exists(p12): if not allow_tcp: self.warning("No gateway PKCS12 specified or file " "not found, gateway disabled: %s", p12) return sec = security.UnsecuredPolicy() self.info("Setting up TCP gateway on ports %d-%d", range[0], range[-1]) else: fac = security.ServerContextFactory(p12_filename=p12, verify_ca_from_p12=True, enforce_cert=True) sec = security.ServerPolicy(fac) self.info("Setting up SSL gateway on ports %d-%d " "using PKCS12 %r", range[0], range[-1], p12) return gateway.Gateway(self, range, hostname=self.get_hostname(), security_policy=sec, log_keeper=self) except Exception as e: error.handle_exception(self, e, "Failed to setup gateway")
def _initiate_tunneling(self, config): try: host = config["host"] port = int(config["port"]) p12 = config["p12"] port_range = range(port, port + TUNNELING_PORT_COUNT) self.info("Setting up tunneling on %s ports %d-%d " "using PKCS12 %r", host, port_range[0], port_range[-1], p12) csec = security.ClientContextFactory(p12_filename=p12, verify_ca_from_p12=True) cpol = security.ClientPolicy(csec) ssec = security.ServerContextFactory(p12_filename=p12, verify_ca_from_p12=True) spol = security.ServerPolicy(ssec) backend = tunneling.Backend(host, port_range, client_security_policy=cpol, server_security_policy=spol) backend.redirect_log(self) frontend = tunneling.Tunneling(backend) return frontend except Exception as e: msg = "Failed to setup tunneling backend" error.handle_exception(self, e, msg) return None
def on_become_master(self): self._ssh.start_listening() self._journaler.set_connection_strings( self.config.agency.journal) try: self._start_master_gateway() except Exception as e: error.handle_exception( self, e, "Failed setting up gateway, it will stay disabled.") self._redirect_text_log() self._create_pid_file() self._link_log_file(options.MASTER_LOG_LINK) signal.signal(signal.SIGUSR1, self._sigusr1_handler) signal.signal(signal.SIGUSR2, self._sigusr2_handler) backends = [] backends.append(self._initiate_messaging(self.config.msg)) backends.append(self._initiate_tunneling(self.config.tunnel)) backends.append(unix.Master(self._broker)) backends = filter(None, backends) d = defer.succeed(None) for backend in backends: d.addCallback(defer.drop_param, self._messaging.add_backend, backend) if (self.config.agency.enable_spawning_slave and sys.platform != "win32"): d.addCallback(defer.drop_param, self._spawn_backup_agency) d.addCallback(defer.drop_param, self._start_host_agent) return d
def solve(connection, doc_id): connection.info('Solving conflicts for document: %s', doc_id) plain_doc = yield connection.get_document(doc_id, raw=True, conflicts=True) if '_conflicts' not in plain_doc: connection.debug('Document:%s is not in state conflict, aborting.', doc_id) return doc = connection._unserializer.convert(plain_doc) if not IDocument.providedBy(doc): handler = _solve_alert else: strategy_handlers = { ConflictResolutionStrategy.db_winner: _solve_db_winner, ConflictResolutionStrategy.alert: _solve_alert, ConflictResolutionStrategy.merge: _solve_merge} s = type(doc).conflict_resolution_strategy handler = strategy_handlers.get(s, _solve_alert) connection.debug("Using %s strategy", handler.__name__) try: yield handler(connection, doc, plain_doc['_conflicts']) connection.debug("Solving conflict for document %s completed", doc_id) except UnsolvableConflict: raise except Exception as e: error.handle_exception(None, e, "Failed solving conflict") raise UnsolvableConflict(str(e), doc)
def _create_gateway(self, config): try: port = int(config["port"]) p12 = config["p12"] allow_tcp = config["allow_tcp"] range = (port, port + GATEWAY_PORT_COUNT) if not os.path.exists(p12): if not allow_tcp: self.warning("No gateway PKCS12 specified or file " "not found, gateway disabled: %s", p12) return sec = security.UnsecuredPolicy() self.info("Setting up TCP gateway on ports %d-%d", range[0], range[-1]) else: fac = security.ServerContextFactory(p12_filename=p12, verify_ca_from_p12=True) sec = security.ServerPolicy(fac) self.info("Setting up SSL gateway on ports %d-%d " "using PKCS12 %r", range[0], range[-1], p12) return gateway.Gateway(self, range, security_policy=sec) except Exception as e: error.handle_exception(self, e, "Failed to setup gateway")
def apply(self): try: if self.agent_id != self._replay.agent_id: raise ReplayError( "Tried to apply the entry belonging to the " "agent id: %r, but the Replay instance " "belogs to: %r" % (self.agent_id, self._replay.agent_id) ) self._replay.set_current_time(self._timestamp) if self.journal_id == "agency": self._replay.replay_agency_entry(self) return self self._replay._log_entry(self) self._replay.require_agent() instance = self._replay.registry.get(self.journal_id, None) if instance is None: raise ReplayError( "Instance for journal_id %r not found " "in the registry when replaying %r" % (self.journal_id, self.function_id) ) result = instance.replay(self) if self._next_effect < len(self._side_effects): remaining = self._side_effects[self._next_effect] side_effect = self.restore_side_effect(remaining, parse_args=True) function_id, args, kwargs, _effects, _result = side_effect se_desc = side_effect_as_string(function_id, args, kwargs) raise ReplayError("Function %s did not consume side-effect %s" % (self.function_id, se_desc)) frozen_result = self._replay.serializer.freeze(result) unfrozen_result = self._replay.unserializer.convert(frozen_result) expected = self._replay.unserializer.convert(self.frozen_result) if unfrozen_result != expected: res = pformat(unfrozen_result) exp = pformat(expected) diffs = text_helper.format_diff(exp, res, "\n ") raise ReplayError( "Function %r replay result " "does not match recorded one.\n" " RESULT: %s\n" " EXPECTED: %s\n" " DIFFERENCES: %s\n" % (self.function_id, res, exp, diffs) ) self._replay.log("State after the entry: %r", self._replay.agent._get_state()) return self except Exception as e: error.handle_exception( "replay", e, "Failed trying to apply instance %r entry %r: ", self.journal_id, self.function_id ) raise
def _finish(self): try: if self._cache is not None: data = self._cache.getvalue() self._cache = None self.prepare() self._request._ref.write(self._encode(data)) except Exception, e: msg = "Exception during response finalization" error.handle_exception(self, e, msg)
def _update_xdot(self, model, graph): rect = self._xdot.get_allocation() graph.obj_dict["attributes"]["ratio"] = float(rect.height) / rect.width self._graph = graph try: dotcode = graph.to_string() except Exception as e: error.handle_exception(None, e, "Failed to generate xdot") else: self._xdot.set_dotcode(dotcode) if self._xdot.graph.width: self._xdot.zoom_to_fit()
def unload(name): log.info("application", "Starting unloading application %r", name) r = get_application_registry() application = r.lookup(name) if not application: log.error("application", "Tried to unload application which is not " "loaded: %r", name) return try: application.unload() except Exception as e: error.handle_exception("application", e, "Problem while unloading " "application %r", name) log.info("application", "Unloading application %r complete", name)
def testHandleException(self): xlog.setDebug('4') try: do_raise() except Error as e: error.handle_exception(self.logger, e, "Error handler:") self.assertEqual(1, len(self.keeper.errors)) self.assertEqual(2, len(self.keeper.errors[0][1])) traceback = self.keeper.errors[0][1][1] self.assertIn( 'feat/test/test_common_error.py', traceback)
def do_import(self, iter, force=False): canonical_name, auto = self._parse_row(iter) if force or auto: try: o = reflect.named_object(canonical_name) if IApplication.providedBy(o): o.load() elif isinstance(o, types.ModuleType): reflect.named_module(canonical_name) else: raise TypeError("Uknown type of canonical name target: " "%r -> %s" % (canonical_name, type(o))) except Exception, e: error.handle_exception('imports', e, 'Error importing')
def _initiate_messaging(self, mconfig): assert isinstance(mconfig, config.MsgConfig), str(type(mconfig)) try: host = mconfig.host port = int(mconfig.port) username = mconfig.user password = mconfig.password self.info("Setting up messaging using %s@%s:%d", username, host, port) backend = net.RabbitMQ(host, port, username, password) client = rabbitmq.Client(backend, self.get_hostname()) return client except Exception as e: msg = "Failed to setup messaging backend" error.handle_exception(self, e, msg) # For now we do not support not having messaging backend raise
def _initiate_messaging(self, config): try: host = config['host'] port = int(config['port']) username = config['user'] password = config['password'] self.info("Setting up messaging using %s@%s:%d", username, host, port) backend = messaging.Messaging(host, port, username, password) backend.redirect_log(self) return backend except Exception as e: msg = "Failed to setup messaging backend" error.handle_exception(self, e, msg) # For now we do not support not having messaging backend raise
def validate_html(self): try: if os.environ['SELENIUM_SKIP_HTML_VALIDATION']: return except KeyError: pass url = os.environ.get("SELENIUM_VALIDATOR", 'validator.w3.org') validator = httpclient.Connection(url, 80, logger=self) self.addCleanup(validator.disconnect) source = self.browser.page_source # browser returns the result as ISO-8859-1 encoded, but it's later # interpreted as unicode. Here I just remove all the nonascii # characters from the input, I didn't find the better way to deal # with this. source = "".join(x for x in source if ord(x) < 128) datagen, headers = encode.multipart_encode({ 'fragment': source, 'charset': '(detect automatically)', 'group': 0, 'ss': 1, 'user-agent': 'W3C_Validator/1.3', 'doctype': 'inline'}) body = "".join(datagen) # HttpConnection for some reason uses lowercase headers headers = dict((k.lower(), v) for k, v in headers.iteritems()) self.info("Validating html. Posting to %s/check", validator._host) try: response = yield validator.request( http.Methods.POST, '/check', headers, body) except Exception as e: error.handle_exception(self, e, "Failed posting to validator") self.fail("Failed posting to validator") errors = response.headers.get('x-w3c-validator-errors') if errors != '0': html_name = '%s.html' % (self.browser.title, ) with open(html_name, 'w') as f: f.write(response.body) self.fail("Failing because of invalid html. " "Saved validator output to %s\n" % (html_name, ))
def migration_script(connection): log.info("script", "Running the migration script.") index = yield connection.query_view(view.DocumentByType, group_level=2, parse_result=False) try: for (type_name, version), count in index: restorator = serialization.lookup(type_name) if not restorator: log.error( 'script', "Failed to lookup the restorator for the " "type name: %s. There is %d objects like this in the" " database. They will not be migrated.", type_name, count) if (IVersionAdapter.providedBy(restorator) and ((version is None and restorator.version > 1) or (version is not None and version < restorator.version))): log.info('script', "I will migrate %d documents of the " "type: %s from version %s to %d", count, type_name, version, restorator.version) migrated = 0 while migrated < count: fetched = yield connection.query_view( view.DocumentByType, key=(type_name, version), limit=15, reduce=False, include_docs=True) migrated += len(fetched) if not fetched: break log.info("script", "Migrated %d documents of the type %s " "from %s version to %s", migrated, type_name, version, restorator.version) except Exception: error.handle_exception("script", None, "Failed running migration script") raise
def _emergency_termination(self, failure, request, response): try: msg = "Unrecovered Failure During Web Request" error.handle_failure(self, failure, msg) if response.can_update_headers: response.set_header("content-type", "text/plain") response.write("Error: %s" % failure.getErrorMessage()) if failure.check(http.HTTPError): return self._terminate(request, response, failure.value.status_code) else: return self._terminate(request, response) except Exception, e: msg = "Exception during emergency termination" error.handle_exception(self, e, msg) # We don't call _terminate() here, we already too deep in the mess return None
def terminate_agents(self): self._agency = self.model.view self._agency._starting_host = True application = self.model.source agent_ids = [] for medium in list(self._agency.iter_agents()): if medium.agent.application != application: self._agency.info( "Leaving in peace the agent %s, he belongs to application" " %s", medium.get_agent_type(), medium.agent.application.name) continue a_id = medium.get_agent_id() agent_ids.append(a_id) self._agency.info("Terminating %s with id: %s", medium.get_agent_type(), a_id) try: yield medium.terminate_hard() except Exception as e: error.handle_exception('restart', e, "Error termination.") defer.returnValue(agent_ids)
def wrapped(*args, **kwargs): noncritical = kwargs.pop('noncritical', False) self.logex(5, "%s call: %s, args=%r, kwargs=%r", (self.name, unwrapped.__name__, args, kwargs), depth=-3) try: if self._explicit_wait is None: return self._wrap_call(unwrapped, *args, **kwargs) else: return self._wrap_call(explicitly_wait, unwrapped, args, kwargs, timeout=self._explicit_wait) except Exception as e: error.handle_exception( self, e, "%s call failed, name: %s, args=%r, kwargs=%r", self.name, unwrapped.__name__, args, kwargs) self.on_error(e) if not noncritical: raise else: self.info("Test will continue, as the call was done " "with noncritical=True")
def _remove_file(self, path): try: os.unlink(path) except Exception as e: error.handle_exception(self, e, 'Failed to remove file: %s', path)
def _recorded_call(self, fun_id, function, args, kwargs, reentrant=True): # Starts the fiber section section = fiber.WovenSection() section.enter() fibdesc = section.descriptor # Check if we are in replay mode mode = section.state.get(RECMODE_TAG, None) if mode == JournalMode.replay: fun_id, function = self._resolve_function(fun_id, function) return self._call_fun(fun_id, function, args, kwargs) # Check if this is the first recording in the fiber section recording = section.state.get(RECORDING_TAG, None) section_first = recording is None result = None try: entry = section.state.get(JOURNAL_ENTRY_TAG, None) mode = section.state.get(RECMODE_TAG, None) fiber_first = section_first and section.descriptor.fiber_depth == 0 fun_id, function = self._resolve_function(fun_id, function) if section_first: entry = self.journal_keeper.new_entry(self.journal_id, fun_id, *args, **kwargs) entry.set_fiber_context(fibdesc.fiber_id, fibdesc.fiber_depth) section.state[RECORDING_TAG] = True section.state[RECMODE_TAG] = JournalMode.recording section.state[JOURNAL_ENTRY_TAG] = entry if not (fiber_first or reentrant): # If not reentrant and it is not the first, it's BAAAAAD. raise ReentrantCallError("Recorded functions %s cannot be " "called from inside the recording " "section" % (fun_id, )) result = self._call_fun(fun_id, function, args, kwargs) except Exception as e: if not section_first: raise result = fiber.fail(e) error.handle_exception(self, e, "Exception inside recorded " "function %s", fun_id) finally: if section_first: entry.set_result(result) entry.commit() result = entry.get_result() section.state[RECORDING_TAG] = None section.state[JOURNAL_ENTRY_TAG] = None section.state[RECMODE_TAG] = None return section.exit(result)
def _prepare_error(self, exception, request, response): if isinstance(exception, http.NotAuthorizedError): if response.can_update_headers: challenge = exception.challenge if challenge: header = challenge.header_value response.set_header("WWW-Authenticate", header) response.set_status(http.Status.UNAUTHORIZED) elif isinstance(exception, http.NotAcceptableError): if response.can_update_headers: # Add NON-STANDARD headers to the response if exception.allowed_mime_types: allowed = ", ".join(exception.allowed_mime_types) response.set_header("Allow-Type", allowed) if exception.allowed_encodings: allowed = ", ".join(exception.allowed_encodings) response.set_header("Allow-Charset", allowed) if exception.allowed_languages: allowed = ", ".join(exception.allowed_languages) response.set_header("Allow-Language", allowed) response.set_status(http.Status.NOT_ACCEPTABLE) elif isinstance(exception, http.NotAllowedError): if response.can_update_headers: value = ", ".join([m.name for m in exception.allowed_methods]) response.set_header("Allow", value) response.set_status(http.Status.NOT_ALLOWED) elif isinstance(exception, http.MovedPermanently): if response.can_update_headers: response.set_header("Location", exception.location) response.set_status(http.Status.MOVED_PERMANENTLY) response.set_header("Cache-Control", "no-store") response.set_header("connection", "close") return None elif isinstance(exception, http.HTTPError): if response.can_update_headers: response.set_status(exception.status_code) if not exception.status_code.is_error(): # Not a real error, so we terminate return None elif isinstance(exception, UnicodeEncodeError): # We failed to encode data in the response selected charset # As for HTTP/1.1 we should set the accepted characteristics, # but it would be hard at this point given we don't know what # triggered this exception. msg = "Failed to encode response to accepted charset" error.handle_exception(self, exception, msg) if response.can_update_headers: response.set_status(http.Status.NOT_ACCEPTABLE) else: msg = "Exception during HTTP resource rendering" error.handle_exception(self, exception, msg) if response.can_update_headers: response.set_status(http.Status.INTERNAL_SERVER_ERROR) return exception