def main(unused_argv):
"""Main."""
token = GetToken()
config_lib.CONFIG.AddContext("Commandline Context")
config_lib.CONFIG.AddContext("ConfigUpdater Context")
if flags.FLAGS.subparser_name == "initialize":
startup.ConfigInit()
if flags.FLAGS.noprompt:
InitializeNoPrompt(config_lib.CONFIG, token=token)
else:
Initialize(config_lib.CONFIG, token=token)
return
startup.Init()
try:
print "Using configuration %s" % config_lib.CONFIG
except AttributeError:
raise RuntimeError("No valid config specified.")
if flags.FLAGS.subparser_name == "generate_keys":
try:
GenerateKeys(config_lib.CONFIG,
overwrite_keys=flags.FLAGS.overwrite_keys)
except RuntimeError, e:
# GenerateKeys will raise if keys exist and overwrite_keys is not set.
print "ERROR: %s" % e
sys.exit(1)
config_lib.CONFIG.Write()
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("Worker Context",
"Context applied when running a worker.")
# Initialise flows
startup.Init()
token = access_control.ACLToken(username="******").SetUID()
worker_obj = worker.GRRWorker(token=token)
worker_obj.Run()
def __init__(self):
startup.Init()
self.server_pem = config_lib.CONFIG["Frontend.certificate"]
self.front_end = flow.FrontEndServer(
certificate=config_lib.CONFIG["Frontend.certificate"],
private_key=config_lib.CONFIG["PrivateKeys.server_key"],
max_queue_size=config_lib.CONFIG["Frontend.max_queue_size"],
message_expiry_time=config_lib.CONFIG["Frontend.message_expiry_time"],
max_retransmission_time=config_lib.CONFIG[
"Frontend.max_retransmission_time"])
def main(_):
"""Run the main test harness."""
config_lib.CONFIG.AddContext(
"AdminUI Context",
"Context applied when running the admin user interface GUI.")
startup.Init()
# Start up a server in another thread
bind_address = config_lib.CONFIG["AdminUI.bind"]
ip = ipaddr.IPAddress(bind_address)
if ip.version == 4:
# Address looks like an IPv4 address.
ThreadingDjango.address_family = socket.AF_INET
max_port = config_lib.CONFIG.Get("AdminUI.port_max",
config_lib.CONFIG["AdminUI.port"])
for port in range(config_lib.CONFIG["AdminUI.port"], max_port + 1):
# Make a simple reference implementation WSGI server
try:
server = simple_server.make_server(bind_address,
port,
django_lib.GetWSGIHandler(),
server_class=ThreadingDjango)
break
except socket.error as e:
if e.errno == socket.errno.EADDRINUSE and port < max_port:
logging.info("Port %s in use, trying %s", port, port + 1)
else:
raise
proto = "HTTP"
if config_lib.CONFIG["AdminUI.enable_ssl"]:
cert_file = config_lib.CONFIG["AdminUI.ssl_cert_file"]
if not cert_file:
raise ValueError("Need a valid cert file to enable SSL.")
key_file = config_lib.CONFIG["AdminUI.ssl_key_file"]
server.socket = ssl.wrap_socket(server.socket,
certfile=cert_file,
keyfile=key_file,
server_side=True)
proto = "HTTPS"
# SSL errors are swallowed by the WSGIServer so if your configuration does
# not work, uncomment the line below, point your browser at the gui and look
# at the log file to see why SSL complains:
# server.socket.accept()
sa = server.socket.getsockname()
logging.info("Serving %s on %s port %d ...", proto, sa[0], sa[1])
startup.DropPrivileges()
server.serve_forever()
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("HTTPServer Context")
startup.Init()
httpd = CreateServer()
try:
httpd.serve_forever()
except KeyboardInterrupt:
print "Caught keyboard interrupt, stopping"
def main(argv):
"""Sets up all the component in their own threads."""
flags.FLAGS.config = config_lib.CONFIG["Test.config"]
config_lib.CONFIG.AddContext(
"Demo Context",
"The demo runs all functions in a single process using the "
"in memory data store.")
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
flags.FLAGS.config = config_lib.CONFIG["Test.config"]
flags.FLAGS.secondary_configs = [
os.path.join(config_lib.CONFIG["Test.data_dir"], "grr_test.yaml")
]
startup.Init()
# pylint: disable=unused-import,unused-variable,g-import-not-at-top
from grr.gui import gui_plugins
# pylint: enable=unused-import,unused-variable,g-import-not-at-top
# This is the worker thread.
worker_thread = threading.Thread(target=worker.main,
args=[argv],
name="Worker")
worker_thread.daemon = True
worker_thread.start()
# This is the enroller thread.
enroller_thread = threading.Thread(target=enroller.main,
args=[argv],
name="Enroller")
enroller_thread.daemon = True
enroller_thread.start()
# This is the http server Frontend that clients communicate with.
http_thread = threading.Thread(target=http_server.main,
args=[argv],
name="HTTP Server")
http_thread.daemon = True
http_thread.start()
client_thread = threading.Thread(target=client.main,
args=[argv],
name="Client")
client_thread.daemon = True
client_thread.start()
# The UI is running in the main thread.
runtests.main(argv)
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("Commandline Context",
"Context applied for all command line tools")
startup.Init()
data_store.default_token = access_control.ACLToken(
username=flags.FLAGS.username or getpass.getuser(),
reason=flags.FLAGS.reason or "export")
# If subcommand was specified by the user in the command line,
# corresponding subparser should have initialized "func" argument
# with a corresponding export plugin's Run() function.
flags.FLAGS.func(flags.FLAGS)
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("Worker Context",
"Context applied when running a worker.")
# Initialise flows
startup.Init()
# Start a worker
token = access_control.ACLToken(username="******")
worker_obj = worker.GRRWorker(queue=worker.DEFAULT_WORKER_QUEUE,
token=token)
worker_obj.Run()
def main(unused_argv):
"""Main."""
startup.Init()
filename = flags.FLAGS.filename
if not os.path.exists(filename):
print "File %s does not exist" % filename
return
with aff4.FACTORY.Create(filestore.NSRLFileStore.PATH, "NSRLFileStore",
mode="rw", token=aff4.FACTORY.root_token) as store:
imported = ImportFile(store, filename, flags.FLAGS.start)
data_store.DB.Flush()
print "Imported %d hashes" % imported
def main(unused_argv):
"""Main."""
banner = ("\nWelcome to the GRR console\n"
"Type help<enter> to get help\n\n")
config_lib.CONFIG.AddContext("Commandline Context")
config_lib.CONFIG.AddContext(
"Console Context",
"Context applied when running the console binary.")
startup.Init()
# To make the console easier to use, we make a default token which will be
# used in StartFlow operations.
data_store.default_token = rdfvalue.ACLToken(username=getpass.getuser(),
reason=flags.FLAGS.reason)
locals_vars = {
"hilfe": Help,
"help": Help,
"__name__": "GRR Console",
"l": Lister,
"o": aff4.FACTORY.Open,
# Bring some symbols from other modules into the console's
# namespace.
"StartFlowAndWait": flow_utils.StartFlowAndWait,
"StartFlowAndWorker": debugging.StartFlowAndWorker,
"RunEndToEndTests": end_to_end_tests.RunEndToEndTests,
}
locals_vars.update(globals()) # add global variables to console
if flags.FLAGS.client is not None:
locals_vars["client"], locals_vars["token"] = console_utils.OpenClient(
client_id=flags.FLAGS.client)
if flags.FLAGS.code_to_execute:
logging.info("Running code from flag: %s", flags.FLAGS.code_to_execute)
exec(flags.FLAGS.code_to_execute) # pylint: disable=exec-used
elif flags.FLAGS.command_file:
logging.info("Running code from file: %s", flags.FLAGS.command_file)
execfile(flags.FLAGS.command_file)
if (flags.FLAGS.exit_on_complete and
(flags.FLAGS.code_to_execute or flags.FLAGS.command_file)):
return
else: # We want the normal shell.
locals_vars.update(globals()) # add global variables to console
ipshell.IPShell(argv=[], user_ns=locals_vars, banner=banner)
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext(
"Enroller Context",
"Context applied when running within the enroller process")
# Initialise everything.
startup.Init()
# Start an Enroler.
token = access_control.ACLToken(username="******", reason="Implied.")
enroller = worker.GRREnroler(queue=worker.DEFAULT_ENROLLER_QUEUE,
token=token)
enroller.Run()
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("Worker Context",
"Context applied when running a worker.")
# Initialise flows
startup.Init()
workers = {}
dead_count = 0
start_time = time.time()
if config_lib.CONFIG["Worker.worker_process_count"] <= 1:
# Special case when we only support a single worker process and don't need
# the complexity of multiprocessing.
StartWorker()
else:
# Run a pool of workers, reviving any that die and logging the death.
while True:
# Start any required new workers.
while len(workers
) < config_lib.CONFIG["Worker.worker_process_count"]:
worker_process = multiprocessing.Process(target=StartWorker)
worker_process.start()
logging.debug("Added new worker %d", worker_process.pid)
workers[worker_process.name] = worker_process
# Kill off any dead workers.
dead_workers = []
for worker_name, worker_process in workers.items():
if not worker_process.is_alive():
logging.error("Worker %s is dead", worker_process.pid)
dead_workers.append(worker_name)
dead_count += 1
for worker_name in dead_workers:
del workers[worker_name]
# Catch all workers dying on startup and raise immediately instead of
# continuously respawning them.
if (time.time() - start_time) < 60:
if dead_count >= config_lib.CONFIG[
"Worker.worker_process_count"]:
for worker_process in workers.values():
worker_process.terminate()
raise RuntimeError(
"Workers did not start up, all of them died.")
time.sleep(10)
def AddUsers(token=None):
# Now initialize with our modified config.
startup.Init()
print "\nStep 3: Adding Admin User"
try:
AddUser("admin", labels=["admin"], token=token,
password=flags.FLAGS.admin_password)
except UserError:
if flags.FLAGS.noprompt:
UpdateUser("admin", password=flags.FLAGS.admin_password,
add_labels=["admin"], token=token)
else:
if ((raw_input("User 'admin' already exists, do you want to "
"reset the password? [yN]: ").upper() or "N") == "Y"):
UpdateUser("admin", password=True, add_labels=["admin"], token=token)
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("HTTPServer Context")
startup.Init()
httpd = CreateServer()
if config_lib.CONFIG["Frontend.processes"] > 1:
# Multiprocessing
for _ in range(config_lib.CONFIG["Frontend.processes"] - 1):
Process(target=Serve, args=(httpd, )).start()
try:
httpd.serve_forever()
except KeyboardInterrupt:
print "Caught keyboard interrupt, stopping"
def main(unused_argv):
"""Main."""
config_lib.CONFIG.AddContext("Commandline Context")
config_lib.CONFIG.AddContext("ConfigUpdater Context")
startup.Init()
try:
print "Using configuration %s" % config_lib.CONFIG.parser
except AttributeError:
raise RuntimeError("No valid config specified.")
if flags.FLAGS.subparser_name == "load_memory_drivers":
LoadMemoryDrivers(flags.FLAGS.share_dir)
elif flags.FLAGS.subparser_name == "generate_keys":
try:
GenerateKeys(config_lib.CONFIG)
except RuntimeError, e:
# GenerateKeys will raise if keys exist and --overwrite is not set.
print "ERROR: %s" % e
sys.exit(1)
config_lib.CONFIG.Write()
def main(_):
"""Run the main test harness."""
config_lib.CONFIG.AddContext(
"AdminUI Context",
"Context applied when running the admin user interface GUI.")
startup.Init()
# Start up a server in another thread
# Make a simple reference implementation WSGI server
server = simple_server.make_server(config_lib.CONFIG["AdminUI.bind"],
config_lib.CONFIG["AdminUI.port"],
django_lib.GetWSGIHandler(),
server_class=ThreadingDjango)
proto = "HTTP"
if config_lib.CONFIG["AdminUI.enable_ssl"]:
cert_file = config_lib.CONFIG["AdminUI.ssl_cert_file"]
if not cert_file:
raise ValueError("Need a valid cert file to enable SSL.")
key_file = config_lib.CONFIG["AdminUI.ssl_key_file"]
server.socket = ssl.wrap_socket(server.socket, certfile=cert_file,
keyfile=key_file, server_side=True)
proto = "HTTPS"
# SSL errors are swallowed by the WSGIServer so if your configuration does
# not work, uncomment the line below, point your browser at the gui and look
# at the log file to see why SSL complains:
# server.socket.accept()
sa = server.socket.getsockname()
logging.info("Serving %s on %s port %d ...", proto, sa[0], sa[1])
server.serve_forever()
def main(unused_argv):
config_lib.CONFIG.AddContext(
"Commandline Context",
"Context applied for all command line tools")
startup.Init()
if fuse is None:
logging.critical("""Could not start!
fusepy must be installed to run fuse_mount.py!
Try:
sudo pip install fusepy""")
sys.exit(1)
if not flags.FLAGS.mountpoint:
Usage()
sys.exit(1)
# We multiple inherit from GRRFuse and fuse.Operations. In the
# case that fuse is present, we run the actual FUSE layer, since we have
# fuse.Operations. In the case that fuse is not present, we have already
# exited by now if we were run from the command line, and if we were not run
# from the command line, we've been imported, and we run the tests using a
# mock fuse.
class FuseOperation(GRRFuse, fuse.Operations):
pass
root = flags.FLAGS.aff4path
username = flags.FLAGS.username or getpass.getuser()
token = access_control.ACLToken(username=username,
reason=flags.FLAGS.reason or "fusemount")
# If we're exporting a path inside a client, check to see if we have access to
# that client and get the appropriate token.
client_id = client.GetClientURNFromPath(root)
if client_id is not None:
token = security.Approval.GetApprovalForObject(
client_id,
token=token,
username=username)
data_store.default_token = token
logging.info("fuse_mount.py is mounting %s at %s....", root,
flags.FLAGS.mountpoint)
refresh_policy = flags.FLAGS.refresh_policy
if refresh_policy == "always":
max_age_before_refresh = datetime.timedelta(0)
elif refresh_policy == "never":
# Set the max age to be the maximum possible time difference.
max_age_before_refresh = datetime.timedelta.max
elif refresh_policy == "if_older_than_max_age":
max_age_before_refresh = datetime.timedelta(
seconds=flags.FLAGS.max_age_before_refresh)
else:
# Otherwise, a flag outside the enum was given and the flag validator threw
# an execption.
pass
fuse_operation = FuseOperation(
root=root,
token=token,
max_age_before_refresh=max_age_before_refresh,
ignore_cache=flags.FLAGS.ignore_cache,
force_sparse_image=flags.FLAGS.force_sparse_image,
sparse_image_threshold=flags.FLAGS.sparse_image_threshold,
timeout=flags.FLAGS.timeout)
fuse.FUSE(fuse_operation, flags.FLAGS.mountpoint,
foreground=not flags.FLAGS.background)
def RunEndToEndTests():
runner = unittest.TextTestRunner()
# We are running a test so let the config system know that.
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
startup.Init()
token = access_control.ACLToken(username="******",
reason="Running end to end client tests.")
client_id_set = base.GetClientTestTargets(
client_ids=flags.FLAGS.client_ids,
hostnames=flags.FLAGS.hostnames,
checkin_duration_threshold="1h")
for cls in base.ClientTestBase.classes.values():
for p in cls.platforms:
if p not in set(["Linux", "Darwin", "Windows"]):
raise ValueError("Unsupported platform: %s in class %s" %
(p, cls.__name__))
if not client_id_set:
print(
"No clients to test on. Define Test.end_to_end_client* config "
"options, or pass them as parameters.")
for client in aff4.FACTORY.MultiOpen(client_id_set, token=token):
client = client.Get(client.SchemaCls.SUMMARY)
if hasattr(client, "system_info"):
sysinfo = client.system_info
else:
raise RuntimeError(
"Unknown system type, likely waiting on interrogate"
" to complete.")
for cls in base.ClientTestBase.classes.values():
if flags.FLAGS.testnames and (cls.__name__
not in flags.FLAGS.testnames):
continue
if not aff4.issubclass(cls, base.ClientTestBase):
continue
# Fix the call method so we can use the test runner. See doco in
# base.ClientTestBase
def _RealCall(testcase, *args, **kwds):
return testcase.run(*args, **kwds)
cls.__call__ = _RealCall
if sysinfo.system in cls.platforms:
print "Running %s on %s (%s: %s, %s, %s)" % (
cls.__name__, client.client_id, sysinfo.fqdn,
sysinfo.system, sysinfo.version, sysinfo.machine)
try:
# Mixin the unittest framework so we can use the test runner to run
# the test and get nice output. We don't want to depend on unitttest
# code in the tests themselves.
testcase = cls(client_id=client.client_id,
platform=sysinfo.system,
token=token,
local_client=flags.FLAGS.local_client,
local_worker=flags.FLAGS.local_worker)
runner.run(testcase)
except Exception: # pylint: disable=broad-except
logging.exception("Failed to run test %s", cls)
def main(argv):
"""Sets up all the component in their own threads."""
flag_list = [
flags.FLAGS.start_worker, flags.FLAGS.start_ui,
flags.FLAGS.start_http_server, flags.FLAGS.start_enroller
]
enabled_flags = [f for f in flag_list if f]
# If no start preferences were provided start everything.
if not enabled_flags:
flags.FLAGS.start_worker = True
flags.FLAGS.start_enroller = True
flags.FLAGS.start_http_server = True
flags.FLAGS.start_ui = True
if len(enabled_flags) != 1:
# If we only have one flag, we are running in single component mode and we
# want the component to do the initialization. Otherwise we initialize as
# a SingleServer.
config_lib.CONFIG.AddContext(
"SingleServer Context",
"Context applied when running all functions in a single server.")
startup.Init()
# Start the worker thread if necessary.
if flags.FLAGS.start_worker:
worker_thread = threading.Thread(target=worker.main,
args=[argv],
name="Worker")
worker_thread.daemon = True
worker_thread.start()
# Start the enroller thread if necessary.
if flags.FLAGS.start_enroller:
enroller_thread = threading.Thread(target=enroller.main,
args=[argv],
name="Enroller")
enroller_thread.daemon = True
enroller_thread.start()
# Start the HTTP server thread, that clients communicate with, if necessary.
if flags.FLAGS.start_http_server:
http_thread = threading.Thread(target=http_server.main,
args=[argv],
name="HTTP Server")
http_thread.daemon = True
http_thread.start()
# Start the UI thread if necessary.
if flags.FLAGS.start_ui:
ui_thread = threading.Thread(target=admin_ui.main,
args=[argv],
name="GUI")
ui_thread.daemon = True
ui_thread.start()
try:
while True:
time.sleep(100)
except KeyboardInterrupt:
pass
def RunEndToEndTests():
runner = unittest.TextTestRunner()
# We are running a test so let the config system know that.
config_lib.CONFIG.AddContext("Test Context",
"Context applied when we run tests.")
startup.Init()
token = access_control.ACLToken(username="******",
reason="Running end to end client tests.")
# We need this for the launchbinary test
with aff4.FACTORY.Create("aff4:/users/GRREndToEndTest",
"GRRUser",
mode="rw",
token=token) as test_user:
test_user.AddLabels("admin")
client_id_set = base.GetClientTestTargets(
client_ids=flags.FLAGS.client_ids,
hostnames=flags.FLAGS.hostnames,
checkin_duration_threshold="1h",
token=token)
for cls in base.ClientTestBase.classes.values():
for p in cls.platforms:
if p not in set(["Linux", "Darwin", "Windows"]):
raise ValueError("Unsupported platform: %s in class %s" %
(p, cls.__name__))
if not client_id_set:
print(
"No clients to test on. Define Test.end_to_end_client* config "
"options, or pass them as parameters.")
results_by_client = {}
for client in aff4.FACTORY.MultiOpen(client_id_set, token=token):
client_summary = client.GetSummary()
if hasattr(client_summary, "system_info"):
sysinfo = client_summary.system_info
else:
raise RuntimeError(
"Unknown system type, likely waiting on interrogate"
" to complete.")
results = {}
results_by_client[client.urn] = results
for cls in base.ClientTestBase.classes.values():
if flags.FLAGS.testnames and (cls.__name__
not in flags.FLAGS.testnames):
continue
if not aff4.issubclass(cls, base.ClientTestBase):
continue
if cls.__name__.startswith("Abstract"):
continue
# Fix the call method so we can use the test runner. See doco in
# base.ClientTestBase
def _RealCall(testcase, *args, **kwds):
return testcase.run(*args, **kwds)
cls.__call__ = _RealCall
if sysinfo.system in cls.platforms:
print "Running %s on %s (%s: %s, %s, %s)" % (
cls.__name__, client_summary.client_id, sysinfo.fqdn,
sysinfo.system, sysinfo.version, sysinfo.machine)
try:
# Mixin the unittest framework so we can use the test runner to run
# the test and get nice output. We don't want to depend on unitttest
# code in the tests themselves.
testcase = cls(client_id=client_summary.client_id,
platform=sysinfo.system,
token=token,
local_client=flags.FLAGS.local_client,
local_worker=flags.FLAGS.local_worker)
results[cls.__name__] = runner.run(testcase)
except Exception: # pylint: disable=broad-except
logging.exception("Failed to run test %s", cls)
# Print a little summary.
for client, results in results_by_client.iteritems():
print "Results for %s:" % client
for testcase, result in sorted(results.items()):
res = "[ OK ]"
if result.errors or result.failures:
res = "[ FAIL ]"
print "%45s: %s" % (testcase, res)
def main(argv):
"""Sets up all the component in their own threads."""
flag_list = [flags.FLAGS.start_worker, flags.FLAGS.start_ui,
flags.FLAGS.start_http_server, flags.FLAGS.start_dataserver]
enabled_flags = [f for f in flag_list if f]
# If no start preferences were provided start everything
if not enabled_flags:
flags.FLAGS.start_worker = True
flags.FLAGS.start_http_server = True
flags.FLAGS.start_ui = True
threads = []
if len(enabled_flags) != 1:
# If we only have one flag, we are running in single component mode and we
# want the component to do the initialization. Otherwise we initialize as
# a SingleServer.
config_lib.CONFIG.AddContext(
"SingleServer Context",
"Context applied when running all functions in a single server.")
startup.Init()
# Start the worker thread if necessary.
if flags.FLAGS.start_worker:
worker_thread = threading.Thread(target=worker.main, args=[argv],
name="Worker")
worker_thread.daemon = True
threads.append(worker_thread)
worker_thread.start()
# Start the HTTP server thread, that clients communicate with, if necessary.
if flags.FLAGS.start_http_server:
http_thread = threading.Thread(target=http_server.main, args=[argv],
name="HTTP Server")
http_thread.daemon = True
threads.append(http_thread)
http_thread.start()
# Start the UI thread if necessary.
if flags.FLAGS.start_ui:
ui_thread = threading.Thread(target=admin_ui.main, args=[argv],
name="GUI")
ui_thread.daemon = True
threads.append(ui_thread)
ui_thread.start()
# Start the data server thread if necessary.
if flags.FLAGS.start_dataserver:
dataserver_thread = threading.Thread(target=data_server.main, args=[argv],
name="Dataserver")
dataserver_thread.daemon = True
threads.append(dataserver_thread)
dataserver_thread.start()
try:
while True:
time.sleep(5)
# If any threads die GRR will not work, so if there is a dead one we exit
for thread in threads:
if not thread.is_alive():
raise RuntimeError("Child thread %s has died, exiting" % thread.name)
except KeyboardInterrupt:
pass
