def get_png(script): gnuplot = subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = gnuplot.communicate(script) if output[1]: log.error('gnuplot', "Failed to generate the graph. " "Error returned by gnuplot: \n%s", output[1]) return output[0]
def body(connection): if len(args) < 1: log.error("script", "USAGE: locate.py <agent_id>") return agent_id = args[0] try: host = yield locate(connection, agent_id) except Exception as e: log.error("script", "ERROR: %r", e) log.info("script", "Agent runs at host: %r", host)
def push_initial_data(connection): global _documents for doc in _documents: try: yield connection.save_document(doc) except ConflictError: log.error("script", "Document with id %s already exists!", doc.doc_id) design = view.generate_design_doc() yield connection.save_document(design)
def unload(name): log.info("application", "Starting unloading application %r", name) r = get_application_registry() application = r.lookup(name) if not application: log.error("application", "Tried to unload application which is not " "loaded: %r", name) return try: application.unload() except Exception as e: error.handle_exception("application", e, "Problem while unloading " "application %r", name) log.info("application", "Unloading application %r complete", name)
def get_png(script): gnuplot = subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = gnuplot.communicate(script) if output[1]: log.error( 'gnuplot', "Failed to generate the graph. " "Error returned by gnuplot: \n%s", output[1]) return output[0]
def testDefaultLogging(self): keeper = DummyLogKeeper() log.set_default(keeper) log.log("foo", "1") log.debug("bar", "2", 42) log.info("spam", "3") log.warning("bacon", "4", 2, 3, 5) log.error("eggs", "4") self.assertEqual(keeper.entries, [(LogLevel.log, None, 'foo', '1', (), 1), (LogLevel.debug, None, 'bar', '2', (42, ), 1), (LogLevel.info, None, 'spam', '3', (), 1), (LogLevel.warning, None, 'bacon', '4', (2, 3, 5), 1), (LogLevel.error, None, 'eggs', '4', (), 1)])
def push_initial_data(connection, overwrite=False): documents = applications.get_initial_data_registry().itervalues() for doc in documents: try: yield connection.save_document(doc) except ConflictError: if not overwrite: log.error('script', 'Document with id %s already exists!', doc.doc_id) else: yield _update_old(connection, doc) design_docs = view.generate_design_docs() for design_doc in design_docs: try: yield connection.save_document(design_doc) except ConflictError: yield _update_old(connection, design_doc)
def migration_script(connection): log.info("script", "Running the migration script.") index = yield connection.query_view(view.DocumentByType, group_level=2, parse_result=False) try: for (type_name, version), count in index: restorator = serialization.lookup(type_name) if not restorator: log.error( 'script', "Failed to lookup the restorator for the " "type name: %s. There is %d objects like this in the" " database. They will not be migrated.", type_name, count) if (IVersionAdapter.providedBy(restorator) and ((version is None and restorator.version > 1) or (version is not None and version < restorator.version))): log.info('script', "I will migrate %d documents of the " "type: %s from version %s to %d", count, type_name, version, restorator.version) migrated = 0 while migrated < count: fetched = yield connection.query_view( view.DocumentByType, key=(type_name, version), limit=15, reduce=False, include_docs=True) migrated += len(fetched) if not fetched: break log.info("script", "Migrated %d documents of the type %s " "from %s version to %s", migrated, type_name, version, restorator.version) except Exception: error.handle_exception("script", None, "Failed running migration script") raise
def parse_response(response, tag): if response.status < 300: if (response.headers.get('content-type') == 'application/json'): try: return json.loads(response.body) except ValueError: log.error('couchdb', "Could not parse json data from couchdb. Data: %r", response.body) return failure.Failure(DatabaseError("Json parse error")) else: return response.body else: msg = ("%s gave %s status with body: %s" % (tag, response.status.name, response.body)) if response.status == http.Status.NOT_FOUND: return failure.Failure(NotFoundError(msg)) elif response.status == http.Status.CONFLICT: return failure.Failure(ConflictError(msg)) else: return failure.Failure(DatabaseError(msg))
def testDefaultLogging(self): keeper = DummyLogKeeper() current = log.get_default() log.set_default(keeper) self.addCleanup(log.set_default, current) log.log("foo", "1") log.debug("bar", "2", 42) log.info("spam", "3") log.warning("bacon", "4", 2, 3, 5) log.error("eggs", "4") self.assertEqual( keeper.entries, [ (LogLevel.log, None, "foo", "1", (), 1), (LogLevel.debug, None, "bar", "2", (42,), 1), (LogLevel.info, None, "spam", "3", (), 1), (LogLevel.warning, None, "bacon", "4", (2, 3, 5), 1), (LogLevel.error, None, "eggs", "4", (), 1), ], )
def clean_all_descriptors(connection, dry_run=False): rows = yield connection.query_view(view.DocumentByType, group_level=1, parse_results=False) to_delete = list() for row in rows: type_name = row[0][0] restorator = serialization.lookup(type_name) if not restorator: log.info('cleanup', 'Could not lookup restorator for type name: %s. ' 'There is %s documents of this type.', type_name, row[1]) continue if IDescriptor.implementedBy(restorator): log.info('cleanup', 'I will delete %s documents of type name: %s', row[1], type_name) to_delete.append(type_name) if dry_run: log.info("cleanup", "Not deleting anything, this is just a dry run.") return for type_name in to_delete: keys = view.DocumentByType.fetch(type_name) keys['include_docs'] = False rows = yield connection.query_view( view.DocumentByType, parse_results=False, **keys) for (key, value, doc_id) in rows: try: yield connection.update_document(doc_id, update.delete) except Exception as e: log.error("cleanup", "Cannot delete the documents of type %s with ID: %s." " Reason: %s", type_name, doc_id, e)
def bootstrap(parser=None, args=None, descriptors=None): """Bootstrap a feat process, handling command line arguments. @param parser: the option parser to use; more options will be added to the parser; if not specified or None a new one will be created @type parser: optparse.OptionParser or None @param args: the command line arguments to parse; if not specified or None, sys.argv[1:] will be used @type args: [str()] or None @param descriptors: the descriptors of the agent to starts in addition of the host agent; if not specified or None no additional agents will be started @type descriptors: [Descriptor()] or None @return: the deferred of the bootstrap chain @rtype: defer.Deferred()""" tee = log.init() # The purpose of having log buffer here, is to be able to dump the # log lines to a journal after establishing connection with it. # This is done in stage_configure() of net agency Startup procedure. tee.add_keeper('buffer', log.LogBuffer(limit=10000)) # use the resolver from twisted.names instead of the default # the reason for this is that ThreadedResolver behaves strangely # after the reconnection - raises the DNSLookupError for names # which have been resolved while there was no connection resolver.installResolver(reactor) if parser is None: parser = optparse.OptionParser() options.add_options(parser) try: opts, args = check_options(*parser.parse_args(args)) except Exception as e: error.handle_exception('bootstrap', e, "Failed parsing config") sys.exit(1) if opts.standalone: cls = standalone.Agency else: cls = net_agency.Agency config = config_module.Config() config.load(os.environ, opts) agency = cls(config) applications.load('feat.agents.application', 'feat') applications.load('feat.gateway.application', 'featmodels') d = defer.Deferred() reactor.callWhenRunning(d.callback, None) if not opts.standalone: # specific to running normal agency hostdef = opts.hostdef if opts.hostres or opts.hostcat or opts.hostports: from feat.agents.common import host hostdef = host.HostDef() for resdef in opts.hostres: parts = resdef.split(":", 1) name = parts[0] value = 1 if len(parts) > 1: try: value = int(parts[1]) except ValueError: raise OptionError( "Invalid host resource: %s" % resdef), \ None, sys.exc_info()[2] hostdef.resources[name] = value for catdef in opts.hostcat: name, value = check_category(catdef) hostdef.categories[name] = value if opts.hostports: hostdef.ports_ranges = dict() for ports in opts.hostports: group, start, stop = tuple(ports.split(":")) hostdef.ports_ranges[group] = (int(start), int(stop)) agency.set_host_def(hostdef) d.addCallback(defer.drop_param, agency.initiate) for desc, kwargs, name in opts.agents: d.addCallback(defer.drop_param, agency.add_static_agent, desc, kwargs, name) else: # standalone specific kwargs = opts.standalone_kwargs or dict() to_spawn = opts.agent_id or opts.agents[0][0] d.addCallback(defer.drop_param, agency.initiate) d.addCallback(defer.drop_param, agency.spawn_agent, to_spawn, **kwargs) queue = None if opts.agency_daemonize: import multiprocessing queue = multiprocessing.Queue() d.addCallbacks(_bootstrap_success, _bootstrap_failure, callbackArgs=(queue, ), errbackArgs=(agency, queue)) if not opts.agency_daemonize: reactor.run() else: logname = "%s.%s.log" % ('feat', agency.agency_id) logfile = os.path.join(config.agency.logdir, logname) log.info("bootstrap", "Daemon processs will be logging to %s", logfile) try: pid = os.fork() except OSError, e: sys.stderr.write("Failed to fork: (%d) %s\n" % (e.errno, e.strerror)) os._exit(1) if pid > 0: # original process waits for information about what status code # to use on exit log.info('bootstrap', "Waiting for deamon process to intialize the agency") try: exit_code, reason = queue.get(timeout=20) except multiprocessing.queues.Empty: log.error('bootstrap', "20 seconds timeout expires waiting for agency" " in child process to initiate.") os._exit(1) else: log.info('bootstrap', "Process exiting with %d status", exit_code) if exit_code: log.info('bootstrap', 'Reason for failure: %s', reason) sys.exit(exit_code) else: # child process performs second fork try: pid = os.fork() except OSError, e: sys.stderr.write("Failed to fork: (%d) %s\n" % (e.errno, e.strerror)) os._exit(1) if pid > 0: # child process just exits sys.exit(0) else: # grandchild runs the reactor and logs to an external log file log.FluLogKeeper.redirect_to(logfile, logfile) reactor.run()