def main(unused_argv): """Main.""" token = GetToken() config_lib.CONFIG.AddContext("Commandline Context") config_lib.CONFIG.AddContext("ConfigUpdater Context") if flags.FLAGS.subparser_name == "initialize": startup.ConfigInit() if flags.FLAGS.noprompt: InitializeNoPrompt(config_lib.CONFIG, token=token) else: Initialize(config_lib.CONFIG, token=token) return startup.Init() try: print "Using configuration %s" % config_lib.CONFIG except AttributeError: raise RuntimeError("No valid config specified.") if flags.FLAGS.subparser_name == "generate_keys": try: GenerateKeys(config_lib.CONFIG, overwrite_keys=flags.FLAGS.overwrite_keys) except RuntimeError, e: # GenerateKeys will raise if keys exist and overwrite_keys is not set. print "ERROR: %s" % e sys.exit(1) config_lib.CONFIG.Write()
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("Worker Context", "Context applied when running a worker.") # Initialise flows startup.Init() token = access_control.ACLToken(username="******").SetUID() worker_obj = worker.GRRWorker(token=token) worker_obj.Run()
def __init__(self): startup.Init() self.server_pem = config_lib.CONFIG["Frontend.certificate"] self.front_end = flow.FrontEndServer( certificate=config_lib.CONFIG["Frontend.certificate"], private_key=config_lib.CONFIG["PrivateKeys.server_key"], max_queue_size=config_lib.CONFIG["Frontend.max_queue_size"], message_expiry_time=config_lib.CONFIG["Frontend.message_expiry_time"], max_retransmission_time=config_lib.CONFIG[ "Frontend.max_retransmission_time"])
def main(_): """Run the main test harness.""" config_lib.CONFIG.AddContext( "AdminUI Context", "Context applied when running the admin user interface GUI.") startup.Init() # Start up a server in another thread bind_address = config_lib.CONFIG["AdminUI.bind"] ip = ipaddr.IPAddress(bind_address) if ip.version == 4: # Address looks like an IPv4 address. ThreadingDjango.address_family = socket.AF_INET max_port = config_lib.CONFIG.Get("AdminUI.port_max", config_lib.CONFIG["AdminUI.port"]) for port in range(config_lib.CONFIG["AdminUI.port"], max_port + 1): # Make a simple reference implementation WSGI server try: server = simple_server.make_server(bind_address, port, django_lib.GetWSGIHandler(), server_class=ThreadingDjango) break except socket.error as e: if e.errno == socket.errno.EADDRINUSE and port < max_port: logging.info("Port %s in use, trying %s", port, port + 1) else: raise proto = "HTTP" if config_lib.CONFIG["AdminUI.enable_ssl"]: cert_file = config_lib.CONFIG["AdminUI.ssl_cert_file"] if not cert_file: raise ValueError("Need a valid cert file to enable SSL.") key_file = config_lib.CONFIG["AdminUI.ssl_key_file"] server.socket = ssl.wrap_socket(server.socket, certfile=cert_file, keyfile=key_file, server_side=True) proto = "HTTPS" # SSL errors are swallowed by the WSGIServer so if your configuration does # not work, uncomment the line below, point your browser at the gui and look # at the log file to see why SSL complains: # server.socket.accept() sa = server.socket.getsockname() logging.info("Serving %s on %s port %d ...", proto, sa[0], sa[1]) startup.DropPrivileges() server.serve_forever()
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("HTTPServer Context") startup.Init() httpd = CreateServer() try: httpd.serve_forever() except KeyboardInterrupt: print "Caught keyboard interrupt, stopping"
def main(argv): """Sets up all the component in their own threads.""" flags.FLAGS.config = config_lib.CONFIG["Test.config"] config_lib.CONFIG.AddContext( "Demo Context", "The demo runs all functions in a single process using the " "in memory data store.") config_lib.CONFIG.AddContext("Test Context", "Context applied when we run tests.") flags.FLAGS.config = config_lib.CONFIG["Test.config"] flags.FLAGS.secondary_configs = [ os.path.join(config_lib.CONFIG["Test.data_dir"], "grr_test.yaml") ] startup.Init() # pylint: disable=unused-import,unused-variable,g-import-not-at-top from grr.gui import gui_plugins # pylint: enable=unused-import,unused-variable,g-import-not-at-top # This is the worker thread. worker_thread = threading.Thread(target=worker.main, args=[argv], name="Worker") worker_thread.daemon = True worker_thread.start() # This is the enroller thread. enroller_thread = threading.Thread(target=enroller.main, args=[argv], name="Enroller") enroller_thread.daemon = True enroller_thread.start() # This is the http server Frontend that clients communicate with. http_thread = threading.Thread(target=http_server.main, args=[argv], name="HTTP Server") http_thread.daemon = True http_thread.start() client_thread = threading.Thread(target=client.main, args=[argv], name="Client") client_thread.daemon = True client_thread.start() # The UI is running in the main thread. runtests.main(argv)
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("Commandline Context", "Context applied for all command line tools") startup.Init() data_store.default_token = access_control.ACLToken( username=flags.FLAGS.username or getpass.getuser(), reason=flags.FLAGS.reason or "export") # If subcommand was specified by the user in the command line, # corresponding subparser should have initialized "func" argument # with a corresponding export plugin's Run() function. flags.FLAGS.func(flags.FLAGS)
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("Worker Context", "Context applied when running a worker.") # Initialise flows startup.Init() # Start a worker token = access_control.ACLToken(username="******") worker_obj = worker.GRRWorker(queue=worker.DEFAULT_WORKER_QUEUE, token=token) worker_obj.Run()
def main(unused_argv): """Main.""" startup.Init() filename = flags.FLAGS.filename if not os.path.exists(filename): print "File %s does not exist" % filename return with aff4.FACTORY.Create(filestore.NSRLFileStore.PATH, "NSRLFileStore", mode="rw", token=aff4.FACTORY.root_token) as store: imported = ImportFile(store, filename, flags.FLAGS.start) data_store.DB.Flush() print "Imported %d hashes" % imported
def main(unused_argv): """Main.""" banner = ("\nWelcome to the GRR console\n" "Type help<enter> to get help\n\n") config_lib.CONFIG.AddContext("Commandline Context") config_lib.CONFIG.AddContext( "Console Context", "Context applied when running the console binary.") startup.Init() # To make the console easier to use, we make a default token which will be # used in StartFlow operations. data_store.default_token = rdfvalue.ACLToken(username=getpass.getuser(), reason=flags.FLAGS.reason) locals_vars = { "hilfe": Help, "help": Help, "__name__": "GRR Console", "l": Lister, "o": aff4.FACTORY.Open, # Bring some symbols from other modules into the console's # namespace. "StartFlowAndWait": flow_utils.StartFlowAndWait, "StartFlowAndWorker": debugging.StartFlowAndWorker, "RunEndToEndTests": end_to_end_tests.RunEndToEndTests, } locals_vars.update(globals()) # add global variables to console if flags.FLAGS.client is not None: locals_vars["client"], locals_vars["token"] = console_utils.OpenClient( client_id=flags.FLAGS.client) if flags.FLAGS.code_to_execute: logging.info("Running code from flag: %s", flags.FLAGS.code_to_execute) exec(flags.FLAGS.code_to_execute) # pylint: disable=exec-used elif flags.FLAGS.command_file: logging.info("Running code from file: %s", flags.FLAGS.command_file) execfile(flags.FLAGS.command_file) if (flags.FLAGS.exit_on_complete and (flags.FLAGS.code_to_execute or flags.FLAGS.command_file)): return else: # We want the normal shell. locals_vars.update(globals()) # add global variables to console ipshell.IPShell(argv=[], user_ns=locals_vars, banner=banner)
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext( "Enroller Context", "Context applied when running within the enroller process") # Initialise everything. startup.Init() # Start an Enroler. token = access_control.ACLToken(username="******", reason="Implied.") enroller = worker.GRREnroler(queue=worker.DEFAULT_ENROLLER_QUEUE, token=token) enroller.Run()
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("Worker Context", "Context applied when running a worker.") # Initialise flows startup.Init() workers = {} dead_count = 0 start_time = time.time() if config_lib.CONFIG["Worker.worker_process_count"] <= 1: # Special case when we only support a single worker process and don't need # the complexity of multiprocessing. StartWorker() else: # Run a pool of workers, reviving any that die and logging the death. while True: # Start any required new workers. while len(workers ) < config_lib.CONFIG["Worker.worker_process_count"]: worker_process = multiprocessing.Process(target=StartWorker) worker_process.start() logging.debug("Added new worker %d", worker_process.pid) workers[worker_process.name] = worker_process # Kill off any dead workers. dead_workers = [] for worker_name, worker_process in workers.items(): if not worker_process.is_alive(): logging.error("Worker %s is dead", worker_process.pid) dead_workers.append(worker_name) dead_count += 1 for worker_name in dead_workers: del workers[worker_name] # Catch all workers dying on startup and raise immediately instead of # continuously respawning them. if (time.time() - start_time) < 60: if dead_count >= config_lib.CONFIG[ "Worker.worker_process_count"]: for worker_process in workers.values(): worker_process.terminate() raise RuntimeError( "Workers did not start up, all of them died.") time.sleep(10)
def AddUsers(token=None): # Now initialize with our modified config. startup.Init() print "\nStep 3: Adding Admin User" try: AddUser("admin", labels=["admin"], token=token, password=flags.FLAGS.admin_password) except UserError: if flags.FLAGS.noprompt: UpdateUser("admin", password=flags.FLAGS.admin_password, add_labels=["admin"], token=token) else: if ((raw_input("User 'admin' already exists, do you want to " "reset the password? [yN]: ").upper() or "N") == "Y"): UpdateUser("admin", password=True, add_labels=["admin"], token=token)
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("HTTPServer Context") startup.Init() httpd = CreateServer() if config_lib.CONFIG["Frontend.processes"] > 1: # Multiprocessing for _ in range(config_lib.CONFIG["Frontend.processes"] - 1): Process(target=Serve, args=(httpd, )).start() try: httpd.serve_forever() except KeyboardInterrupt: print "Caught keyboard interrupt, stopping"
def main(unused_argv): """Main.""" config_lib.CONFIG.AddContext("Commandline Context") config_lib.CONFIG.AddContext("ConfigUpdater Context") startup.Init() try: print "Using configuration %s" % config_lib.CONFIG.parser except AttributeError: raise RuntimeError("No valid config specified.") if flags.FLAGS.subparser_name == "load_memory_drivers": LoadMemoryDrivers(flags.FLAGS.share_dir) elif flags.FLAGS.subparser_name == "generate_keys": try: GenerateKeys(config_lib.CONFIG) except RuntimeError, e: # GenerateKeys will raise if keys exist and --overwrite is not set. print "ERROR: %s" % e sys.exit(1) config_lib.CONFIG.Write()
def main(_): """Run the main test harness.""" config_lib.CONFIG.AddContext( "AdminUI Context", "Context applied when running the admin user interface GUI.") startup.Init() # Start up a server in another thread # Make a simple reference implementation WSGI server server = simple_server.make_server(config_lib.CONFIG["AdminUI.bind"], config_lib.CONFIG["AdminUI.port"], django_lib.GetWSGIHandler(), server_class=ThreadingDjango) proto = "HTTP" if config_lib.CONFIG["AdminUI.enable_ssl"]: cert_file = config_lib.CONFIG["AdminUI.ssl_cert_file"] if not cert_file: raise ValueError("Need a valid cert file to enable SSL.") key_file = config_lib.CONFIG["AdminUI.ssl_key_file"] server.socket = ssl.wrap_socket(server.socket, certfile=cert_file, keyfile=key_file, server_side=True) proto = "HTTPS" # SSL errors are swallowed by the WSGIServer so if your configuration does # not work, uncomment the line below, point your browser at the gui and look # at the log file to see why SSL complains: # server.socket.accept() sa = server.socket.getsockname() logging.info("Serving %s on %s port %d ...", proto, sa[0], sa[1]) server.serve_forever()
def main(unused_argv): config_lib.CONFIG.AddContext( "Commandline Context", "Context applied for all command line tools") startup.Init() if fuse is None: logging.critical("""Could not start! fusepy must be installed to run fuse_mount.py! Try: sudo pip install fusepy""") sys.exit(1) if not flags.FLAGS.mountpoint: Usage() sys.exit(1) # We multiple inherit from GRRFuse and fuse.Operations. In the # case that fuse is present, we run the actual FUSE layer, since we have # fuse.Operations. In the case that fuse is not present, we have already # exited by now if we were run from the command line, and if we were not run # from the command line, we've been imported, and we run the tests using a # mock fuse. class FuseOperation(GRRFuse, fuse.Operations): pass root = flags.FLAGS.aff4path username = flags.FLAGS.username or getpass.getuser() token = access_control.ACLToken(username=username, reason=flags.FLAGS.reason or "fusemount") # If we're exporting a path inside a client, check to see if we have access to # that client and get the appropriate token. client_id = client.GetClientURNFromPath(root) if client_id is not None: token = security.Approval.GetApprovalForObject( client_id, token=token, username=username) data_store.default_token = token logging.info("fuse_mount.py is mounting %s at %s....", root, flags.FLAGS.mountpoint) refresh_policy = flags.FLAGS.refresh_policy if refresh_policy == "always": max_age_before_refresh = datetime.timedelta(0) elif refresh_policy == "never": # Set the max age to be the maximum possible time difference. max_age_before_refresh = datetime.timedelta.max elif refresh_policy == "if_older_than_max_age": max_age_before_refresh = datetime.timedelta( seconds=flags.FLAGS.max_age_before_refresh) else: # Otherwise, a flag outside the enum was given and the flag validator threw # an execption. pass fuse_operation = FuseOperation( root=root, token=token, max_age_before_refresh=max_age_before_refresh, ignore_cache=flags.FLAGS.ignore_cache, force_sparse_image=flags.FLAGS.force_sparse_image, sparse_image_threshold=flags.FLAGS.sparse_image_threshold, timeout=flags.FLAGS.timeout) fuse.FUSE(fuse_operation, flags.FLAGS.mountpoint, foreground=not flags.FLAGS.background)
def RunEndToEndTests(): runner = unittest.TextTestRunner() # We are running a test so let the config system know that. config_lib.CONFIG.AddContext("Test Context", "Context applied when we run tests.") startup.Init() token = access_control.ACLToken(username="******", reason="Running end to end client tests.") client_id_set = base.GetClientTestTargets( client_ids=flags.FLAGS.client_ids, hostnames=flags.FLAGS.hostnames, checkin_duration_threshold="1h") for cls in base.ClientTestBase.classes.values(): for p in cls.platforms: if p not in set(["Linux", "Darwin", "Windows"]): raise ValueError("Unsupported platform: %s in class %s" % (p, cls.__name__)) if not client_id_set: print( "No clients to test on. Define Test.end_to_end_client* config " "options, or pass them as parameters.") for client in aff4.FACTORY.MultiOpen(client_id_set, token=token): client = client.Get(client.SchemaCls.SUMMARY) if hasattr(client, "system_info"): sysinfo = client.system_info else: raise RuntimeError( "Unknown system type, likely waiting on interrogate" " to complete.") for cls in base.ClientTestBase.classes.values(): if flags.FLAGS.testnames and (cls.__name__ not in flags.FLAGS.testnames): continue if not aff4.issubclass(cls, base.ClientTestBase): continue # Fix the call method so we can use the test runner. See doco in # base.ClientTestBase def _RealCall(testcase, *args, **kwds): return testcase.run(*args, **kwds) cls.__call__ = _RealCall if sysinfo.system in cls.platforms: print "Running %s on %s (%s: %s, %s, %s)" % ( cls.__name__, client.client_id, sysinfo.fqdn, sysinfo.system, sysinfo.version, sysinfo.machine) try: # Mixin the unittest framework so we can use the test runner to run # the test and get nice output. We don't want to depend on unitttest # code in the tests themselves. testcase = cls(client_id=client.client_id, platform=sysinfo.system, token=token, local_client=flags.FLAGS.local_client, local_worker=flags.FLAGS.local_worker) runner.run(testcase) except Exception: # pylint: disable=broad-except logging.exception("Failed to run test %s", cls)
def main(argv): """Sets up all the component in their own threads.""" flag_list = [ flags.FLAGS.start_worker, flags.FLAGS.start_ui, flags.FLAGS.start_http_server, flags.FLAGS.start_enroller ] enabled_flags = [f for f in flag_list if f] # If no start preferences were provided start everything. if not enabled_flags: flags.FLAGS.start_worker = True flags.FLAGS.start_enroller = True flags.FLAGS.start_http_server = True flags.FLAGS.start_ui = True if len(enabled_flags) != 1: # If we only have one flag, we are running in single component mode and we # want the component to do the initialization. Otherwise we initialize as # a SingleServer. config_lib.CONFIG.AddContext( "SingleServer Context", "Context applied when running all functions in a single server.") startup.Init() # Start the worker thread if necessary. if flags.FLAGS.start_worker: worker_thread = threading.Thread(target=worker.main, args=[argv], name="Worker") worker_thread.daemon = True worker_thread.start() # Start the enroller thread if necessary. if flags.FLAGS.start_enroller: enroller_thread = threading.Thread(target=enroller.main, args=[argv], name="Enroller") enroller_thread.daemon = True enroller_thread.start() # Start the HTTP server thread, that clients communicate with, if necessary. if flags.FLAGS.start_http_server: http_thread = threading.Thread(target=http_server.main, args=[argv], name="HTTP Server") http_thread.daemon = True http_thread.start() # Start the UI thread if necessary. if flags.FLAGS.start_ui: ui_thread = threading.Thread(target=admin_ui.main, args=[argv], name="GUI") ui_thread.daemon = True ui_thread.start() try: while True: time.sleep(100) except KeyboardInterrupt: pass
def RunEndToEndTests(): runner = unittest.TextTestRunner() # We are running a test so let the config system know that. config_lib.CONFIG.AddContext("Test Context", "Context applied when we run tests.") startup.Init() token = access_control.ACLToken(username="******", reason="Running end to end client tests.") # We need this for the launchbinary test with aff4.FACTORY.Create("aff4:/users/GRREndToEndTest", "GRRUser", mode="rw", token=token) as test_user: test_user.AddLabels("admin") client_id_set = base.GetClientTestTargets( client_ids=flags.FLAGS.client_ids, hostnames=flags.FLAGS.hostnames, checkin_duration_threshold="1h", token=token) for cls in base.ClientTestBase.classes.values(): for p in cls.platforms: if p not in set(["Linux", "Darwin", "Windows"]): raise ValueError("Unsupported platform: %s in class %s" % (p, cls.__name__)) if not client_id_set: print( "No clients to test on. Define Test.end_to_end_client* config " "options, or pass them as parameters.") results_by_client = {} for client in aff4.FACTORY.MultiOpen(client_id_set, token=token): client_summary = client.GetSummary() if hasattr(client_summary, "system_info"): sysinfo = client_summary.system_info else: raise RuntimeError( "Unknown system type, likely waiting on interrogate" " to complete.") results = {} results_by_client[client.urn] = results for cls in base.ClientTestBase.classes.values(): if flags.FLAGS.testnames and (cls.__name__ not in flags.FLAGS.testnames): continue if not aff4.issubclass(cls, base.ClientTestBase): continue if cls.__name__.startswith("Abstract"): continue # Fix the call method so we can use the test runner. See doco in # base.ClientTestBase def _RealCall(testcase, *args, **kwds): return testcase.run(*args, **kwds) cls.__call__ = _RealCall if sysinfo.system in cls.platforms: print "Running %s on %s (%s: %s, %s, %s)" % ( cls.__name__, client_summary.client_id, sysinfo.fqdn, sysinfo.system, sysinfo.version, sysinfo.machine) try: # Mixin the unittest framework so we can use the test runner to run # the test and get nice output. We don't want to depend on unitttest # code in the tests themselves. testcase = cls(client_id=client_summary.client_id, platform=sysinfo.system, token=token, local_client=flags.FLAGS.local_client, local_worker=flags.FLAGS.local_worker) results[cls.__name__] = runner.run(testcase) except Exception: # pylint: disable=broad-except logging.exception("Failed to run test %s", cls) # Print a little summary. for client, results in results_by_client.iteritems(): print "Results for %s:" % client for testcase, result in sorted(results.items()): res = "[ OK ]" if result.errors or result.failures: res = "[ FAIL ]" print "%45s: %s" % (testcase, res)
def main(argv): """Sets up all the component in their own threads.""" flag_list = [flags.FLAGS.start_worker, flags.FLAGS.start_ui, flags.FLAGS.start_http_server, flags.FLAGS.start_dataserver] enabled_flags = [f for f in flag_list if f] # If no start preferences were provided start everything if not enabled_flags: flags.FLAGS.start_worker = True flags.FLAGS.start_http_server = True flags.FLAGS.start_ui = True threads = [] if len(enabled_flags) != 1: # If we only have one flag, we are running in single component mode and we # want the component to do the initialization. Otherwise we initialize as # a SingleServer. config_lib.CONFIG.AddContext( "SingleServer Context", "Context applied when running all functions in a single server.") startup.Init() # Start the worker thread if necessary. if flags.FLAGS.start_worker: worker_thread = threading.Thread(target=worker.main, args=[argv], name="Worker") worker_thread.daemon = True threads.append(worker_thread) worker_thread.start() # Start the HTTP server thread, that clients communicate with, if necessary. if flags.FLAGS.start_http_server: http_thread = threading.Thread(target=http_server.main, args=[argv], name="HTTP Server") http_thread.daemon = True threads.append(http_thread) http_thread.start() # Start the UI thread if necessary. if flags.FLAGS.start_ui: ui_thread = threading.Thread(target=admin_ui.main, args=[argv], name="GUI") ui_thread.daemon = True threads.append(ui_thread) ui_thread.start() # Start the data server thread if necessary. if flags.FLAGS.start_dataserver: dataserver_thread = threading.Thread(target=data_server.main, args=[argv], name="Dataserver") dataserver_thread.daemon = True threads.append(dataserver_thread) dataserver_thread.start() try: while True: time.sleep(5) # If any threads die GRR will not work, so if there is a dead one we exit for thread in threads: if not thread.is_alive(): raise RuntimeError("Child thread %s has died, exiting" % thread.name) except KeyboardInterrupt: pass