import StringIO
import sys
import urlparse
import zipfile
import yaml
import logging
from grr.lib import flags
from grr.lib import lexer
from grr.lib import registry
from grr.lib import type_info
from grr.lib import utils
flags.DEFINE_string("config", None, "Primary Configuration file to use.")
flags.DEFINE_list(
"secondary_configs", [],
"Secondary configuration files to load (These override "
"previous configuration files.).")
flags.DEFINE_bool("config_help", False, "Print help about the configuration.")
flags.DEFINE_list("context", [], "Use these contexts for the config.")
flags.DEFINE_list("plugins", [], "Load these files as additional plugins.")
flags.DEFINE_bool("disallow_missing_config_definitions", False,
"If true, we raise an error on undefined config options.")
from grr.lib import server_plugins
from grr.checks import tests
from grr.client import tests
from grr.gui import tests
from grr.lib import flags
from grr.test_lib import test_lib
from grr.lib import tests
from grr.lib import utils
from grr.parsers import tests
from grr.path_detection import tests
from grr.server import tests
from grr.worker import worker_test
# pylint: enable=unused-import,g-bad-import-order
flags.DEFINE_string("output", None,
"The name of the file we write on (default stderr).")
flags.DEFINE_list("exclude_tests", [],
"A comma-separated list of tests to exclude form running.")
flags.DEFINE_integer("processes", 0,
"Total number of simultaneous tests to run.")
def Red(s):
return "\033[91m %s\033[00m" % s
def Green(s):
return "\033[92m %s\033[00m" % s
from grr.lib.aff4_objects import standard
from grr.lib.rdfvalues import client
# Check if fuse is installed. If it's not, set it to None so we know to mock it
# out later.
try:
# pylint: disable=g-import-not-at-top
import fuse
# pylint: enable=g-import-not-at-top
except (EnvironmentError, ImportError):
# We check for ImportErrors and EnvironmentErrors since submit checks throw an
# EnvironmentError when fuse isn't installed.
fuse = None
flags.DEFINE_string("aff4path", "/",
"Path in AFF4 to use as the root of the filesystem.")
flags.DEFINE_string("mountpoint", None,
"Path to point at which the system should be mounted.")
flags.DEFINE_bool("background", False,
"Whether or not to run the filesystem in the background,"
" not viewing debug information.")
flags.DEFINE_float("timeout", 30,
"How long to poll a flow for before giving up.")
flags.DEFINE_integer("max_age_before_refresh", 60 * 5,
"Measured in seconds. Do a client-side update if it's"
" been this long since we last did one.")
import json
import os
import re
import urlparse
from grr.gui import http_api
from grr.lib import flags
from grr.lib import test_lib
from grr.lib import utils
# To update regression data use the following command line:
# grr/gui/api_renderers_regression_data_generate.py > \
# ./grr/gui/static/angular-components/docs/api-docs-examples.json
flags.DEFINE_string(
"api_regression_data",
"grr/gui/static/angular-components/docs/api-docs-examples.json",
"Base path for GRR static files.")
class ApiCallRendererRegressionTest(test_lib.GRRBaseTest):
"""Base class for API renderers regression tests.
Regression tests are supposed to implement a single abstract Run() method.
In the Run() implementation they're supposed to set up necessary
environment and do a number of Check() calls. Every Check() call fetches
a particular URL and keeps the data. Then, if this test class is used
as part of a test suite, generated data will be compared with ones in
flags.FLAGS.api_regression_data file and exception will be raised if
they're different.
import getpass
# pylint: disable=unused-import,g-bad-import-order
from grr.lib import server_plugins
# pylint: enable=unused-import,g-bad-import-order
from grr.lib import access_control
from grr.lib import config_lib
from grr.lib import data_store
from grr.lib import flags
from grr.lib import server_startup
from grr.tools import export_plugins
DESCRIPTION = "Tool for exporting data from GRR to the outside world."
flags.DEFINE_string("username", None,
"Username to use for export operations authorization.")
flags.DEFINE_string("reason", None,
"Reason to use for export operations authorization.")
def AddPluginsSubparsers():
"""Adds subparsers for all the registered export plugins.
Classes inherited from ExportPlugin define subcommands that will be
recognized by the export tool. For each of the plugins a subparser
is created and then configured with ExportPlugin.ConfigurArgParser()
call.
"""
classes = sorted(export_plugins.plugin.ExportPlugin.classes.itervalues(),
key=lambda cls: cls.name)
from grr.lib import utils
from grr.lib.rdfvalues import data_server as rdf_data_server
from grr.lib.rdfvalues import data_store as rdf_data_store
from grr.server.data_server import auth
from grr.server.data_server import constants
from grr.server.data_server import errors
from grr.server.data_server import master
from grr.server.data_server import rebalance
from grr.server.data_server import store
from grr.server.data_server import utils as sutils
flags.DEFINE_integer("port", None, "Specify the data server port.")
flags.DEFINE_string("path", None, "Specify the data store path.")
flags.DEFINE_bool("master", False, "Mark this data server as the master.")
flags.DEFINE_bool("ipv6", False, "Use IPV6 to accept connections.")
class DataServerHandler(BaseHTTPRequestHandler, object):
"""Handler for HTTP requests to the data server."""
# Data store service.
SERVICE = None
# MASTER is set if this data server is also running as master.
MASTER = None
# Set if the server is not the master.
DATA_SERVER = None
from grr.lib import utils
from grr.lib.aff4_objects import security
# Check if fuse is installed. If it's not, set it to None so we know to mock it
# out later.
try:
# pylint: disable=g-import-not-at-top
import fuse
# pylint: enable=g-import-not-at-top
except (EnvironmentError, ImportError):
# We check for ImportErrors and EnvironmentErrors since submit checks throw an
# EnvironmentError when fuse isn't installed.
fuse = None
flags.DEFINE_string("aff4path", None,
"Path in AFF4 to use as the root of the filesystem.")
flags.DEFINE_string("mountpoint", None,
"Path to point at which the system should be mounted.")
flags.DEFINE_bool("background", False,
"Whether or not to run the filesystem in the background,"
"not viewing debug information.")
flags.DEFINE_float("timeout", 30,
"How long to poll a flow for before giving up.")
flags.DEFINE_integer("max_age_before_refresh", 60*5,
"Measured in seconds. Do a client-side update if it's "
"been this long since we last did one.")
from grr.server.grr_response_server.flows.general import processes
from grr.server.grr_response_server.flows.general import transfer
from grr.server.grr_response_server.gui import api_auth_manager
from grr.server.grr_response_server.gui import api_call_router_with_approval_checks
from grr.server.grr_response_server.gui import webauth
from grr.server.grr_response_server.gui import wsgiapp_testlib
from grr.server.grr_response_server.hunts import implementation
from grr.server.grr_response_server.hunts import standard
from grr.test_lib import acl_test_lib
from grr.test_lib import action_mocks
from grr.test_lib import artifact_test_lib
from grr.test_lib import hunt_test_lib
from grr.test_lib import test_lib
flags.DEFINE_string(
"chrome_driver_path", None,
"Path to the chrome driver binary. If not set, webdriver "
"will search on PATH for the binary.")
flags.DEFINE_bool(
"use_headless_chrome", False, "If set, run Chrome driver in "
"headless mode. Useful when running tests in a window-manager-less "
"environment.")
# A increasing sequence of times.
TIME_0 = test_lib.FIXED_TIME
TIME_1 = TIME_0 + rdfvalue.Duration("1d")
TIME_2 = TIME_1 + rdfvalue.Duration("1d")
def DateString(t):
return t.Format("%Y-%m-%d")
from grr.lib import flags
from grr.lib import registry
from grr.lib import utils
from grr.server.grr_response_server.gui import api_auth_manager
# This import guarantees that all API-related RDF types will get imported
# (as they're all references by api_call_router).
# pylint: disable=unused-import
from grr.server.grr_response_server.gui import api_call_router
# pylint: enable=unused-import
from grr.server.grr_response_server.gui import api_regression_http
from grr.server.grr_response_server.gui import webauth
from grr.test_lib import test_lib
flags.DEFINE_string(
"generate", "",
"Generate golden regression data for tests using a given connection type.")
class ApiRegressionTestMetaclass(registry.MetaclassRegistry):
"""Automatica test classes generation through a metaclass."""
connection_mixins = {}
@classmethod
def RegisterConnectionMixin(cls, mixin):
cls.connection_mixins[mixin.connection_type] = mixin
def __init__(cls, name, bases, env_dict): # pylint: disable=no-self-argument
registry.MetaclassRegistry.__init__(cls, name, bases, env_dict)
from grr import config
from grr_api_client import api
from grr.config import contexts
from grr.lib import config_lib
from grr.lib import flags
from grr.server.grr_response_server import access_control
from grr.server.grr_response_server import data_store
from grr.server.grr_response_server import maintenance_utils
from grr.server.grr_response_server import server_startup
from grr_response_test.end_to_end_tests import test_base
# pylint: disable=unused-import
from grr_response_test.end_to_end_tests import tests
# pylint: enable=unused-import
flags.DEFINE_string("api_endpoint", "http://localhost:8000",
"GRR API endpoint.")
flags.DEFINE_string("api_user", "admin", "Username for GRR API.")
flags.DEFINE_string("api_password", "", "Password for GRR API.")
flags.DEFINE_list(
"client_ids", [], "List of client ids to test. If unset we use "
"Test.end_to_end_client_ids from the config.")
flags.DEFINE_list(
"hostnames", [], "List of client hostnames to test. If unset we use "
"Test.end_to_end_client_hostnames from the config.")
flags.DEFINE_list(
"testnames", [], "List of test cases to run. If unset we run all "
from grr.config import contexts
from grr.lib import config_lib
from grr.lib import flags
from grr.lib import registry
from grr.lib import stats
from grr.server import aff4
from grr.server import data_store
from grr.server import server_logging
from grr.server.blob_stores import memory_stream_bs
from grr.server.data_stores import fake_data_store
# Make sure we do not reinitialize multiple times.
INIT_RAN = False
flags.DEFINE_string(
"test_data_store", None,
"The data store implementation to use for running "
"the tests.")
def TestInit():
"""Only used in tests and will rerun all the hooks to create a clean state."""
global INIT_RAN
if stats.STATS is None:
stats.STATS = stats.StatsCollector()
# Tests use both the server template grr_server.yaml as a primary config file
# (this file does not contain all required options, e.g. private keys), and
# additional configuration in test_data/grr_test.yaml which contains typical
# values for a complete installation.
flags.FLAGS.config = config_lib.Resource().Filter(
"""This is a single binary that runs all the GRR components.
To use this entry point you must run "grr_config_updater initialize" first.
"""
# pylint: disable=unused-import,g-bad-import-order
from grr.lib import server_plugins
# pylint: enable=unused-import,g-bad-import-order
from grr.gui import admin_ui
from grr.lib import flags
from grr.server.data_server import data_server
from grr.tools import http_server
from grr.worker import worker
flags.DEFINE_string("component", None,
"Component to start: [http_server|ui|worker|dataserver].")
def main(argv):
"""Sets up all the component in their own threads."""
# We use .startswith so that multiple copies of services can easily be
# created using systemd as worker1 worker2 ... worker25 etc.
# Start as a worker.
if flags.FLAGS.component.startswith("worker"):
worker.main([argv])
# Start as a HTTP server that clients communicate with.
elif flags.FLAGS.component.startswith("http_server"):
http_server.main([argv])
# pylint: disable=unused-import,g-bad-import-order
from grr.lib import server_plugins
from grr.client import tests
# pylint: enable=unused-import,g-bad-import-order
# These need to register plugins so, pylint: disable=unused-import
from grr.lib import flags
from grr.lib import test_lib
from grr.lib import tests
from grr.lib import utils
from grr.parsers import tests
from grr.tools.export_plugins import tests
from grr.worker import worker_test
flags.DEFINE_string("output", None,
"The name of the file we write on (default stderr).")
flags.DEFINE_integer("processes", 5,
"Total number of simultaneous tests to run.")
flags.DEFINE_string("type", "normal",
"The type of the tests to run (normal, benchmarks).")
class Colorizer(object):
"""A Class which wraps a string with colors."""
COLORS = "BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE"
COLOR_MAP = dict([(x, i) for i, x in enumerate(COLORS.split())])
terminal_capable = False
import logging
# We need to import the server_plugins module before other server init modules.
# pylint: disable=unused-import,g-bad-import-order
from grr.server.grr_response_server import server_plugins
# pylint: disable=unused-import,g-bad-import-order
from grr import config
from grr.config import contexts
from grr.lib import flags
from grr.server.grr_response_server import access_control
from grr.server.grr_response_server import data_store
from grr.server.grr_response_server import server_startup
from grr_response_test.end_to_end_tests import runner
flags.DEFINE_string("api_endpoint", "http://localhost:8000",
"GRR API endpoint.")
flags.DEFINE_string("api_user", "admin", "Username for GRR API.")
flags.DEFINE_string("api_password", "", "Password for GRR API.")
flags.DEFINE_string("client_id", "", "Id for client to run tests against.")
flags.DEFINE_list(
"whitelisted_tests", [],
"(Optional) comma-separated list of tests to run (skipping all others).")
flags.DEFINE_list(
"blacklisted_tests", [],
"(Optional) comma-separated list of tests to skip. Trumps "
"--whitelisted_tests if there are any conflicts.")
"""This is a single binary that runs all the GRR components.
To use this entry point you must run "grr_config_updater initialize" first.
"""
# pylint: disable=unused-import,g-bad-import-order
from grr.lib import server_plugins
# pylint: enable=unused-import,g-bad-import-order
from grr.gui import admin_ui
from grr.lib import flags
from grr.tools import frontend
from grr.worker import worker
flags.DEFINE_string(
"component", None,
"Component to start: [frontend|admin_ui|worker|dataserver].")
def main(argv):
"""Sets up all the component in their own threads."""
# We use .startswith so that multiple copies of services can easily be
# created using systemd as worker1 worker2 ... worker25 etc.
# Start as a worker.
if flags.FLAGS.component.startswith("worker"):
worker.main([argv])
# Start as a frontend that clients communicate with.
elif flags.FLAGS.component.startswith("frontend"):
from grr.client import client_plugins
# pylint: enable=unused-import
from grr.client import comms
from grr.client import vfs
from grr.lib import config_lib
from grr.lib import flags
from grr.lib import startup
from grr.lib.rdfvalues import crypto as rdf_crypto
from grr.lib.rdfvalues import paths as rdf_paths
flags.DEFINE_integer("nrclients", 1,
"Number of clients to start")
flags.DEFINE_string("cert_file", "",
"Path to a file that stores all certificates for"
"the client pool.")
flags.DEFINE_bool("enroll_only", False,
"If specified, the script will enroll all clients and exit.")
class PoolGRRClient(client.GRRClient, threading.Thread):
"""A GRR client for running in pool mode."""
def __init__(self, *args, **kw):
"""Constructor."""
super(PoolGRRClient, self).__init__(*args, **kw)
self.daemon = True
self.stop = False
from grr.server.flows.general import discovery
from grr.server.hunts import results as hunts_results
from grr.test_lib import testing_startup
flags.DEFINE_list("tests",
None,
help=("Test module to run. If not specified we run"
"All modules in the test suite."))
flags.DEFINE_list("labels", ["small"],
"A list of test labels to run. (e.g. benchmarks,small).")
flags.DEFINE_string(
"profile", "", "If set, the test is run using cProfile, storing the "
"resulting profile data in the given filename. Running multiple tests"
"will overwrite this file so --tests should be used to limit this to a"
"single test.")
FIXED_TIME = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("8d")
TEST_CLIENT_ID = rdf_client.ClientURN("C.1000000000000000")
class Error(Exception):
"""Test base error."""
class GRRBaseTest(unittest.TestCase):
"""This is the base class for all GRR tests."""
__metaclass__ = registry.MetaclassRegistry
import csv
import os
# pylint: disable=unused-import,g-bad-import-order
from grr.lib import server_plugins
# pylint: enable=unused-import,g-bad-import-order
from grr.lib import flags
from grr.lib import utils
from grr.server import aff4
from grr.server import data_store
from grr.server import server_startup
from grr.server.aff4_objects import filestore
flags.DEFINE_string("filename", "", "File with hashes.")
flags.DEFINE_integer("start", None, "Start row in the file.")
def _ImportRow(store, row, product_code_list, op_system_code_list):
sha1 = row[0].lower()
md5 = row[1].lower()
crc = int(row[2].lower(), 16)
file_name = utils.SmartUnicode(row[3])
file_size = int(row[4])
special_code = row[7]
store.AddHash(sha1, md5, crc, file_name, file_size, product_code_list,
op_system_code_list, special_code)
def ImportFile(store, filename, start):
from grr.lib.aff4_objects import security
# All the functions in this lib we want in local namespace.
# pylint: disable=wildcard-import
from grr.lib.console_utils import *
# pylint: enable=wildcard-import
from grr.lib.flows import console
from grr.lib.flows.console import debugging
from grr.lib.flows.general import memory
# pylint: enable=unused-import
from grr.tools import end_to_end_tests
flags.DEFINE_string(
"client", None, "Initialise the console with this client id "
"(e.g. C.1234345).")
flags.DEFINE_string("reason", None,
"Create a default token with this access reason ")
flags.DEFINE_string(
"code_to_execute", None,
"If present, no console is started but the code given in "
"the flag is run instead (comparable to the -c option of "
"IPython).")
flags.DEFINE_string(
"command_file", None,
"If present, no console is started but the code given in "
"command file is supplied as input instead.")
from grr_api_client import api_shell_lib
from grr.config import contexts
from grr.config import server as config_server
from grr.lib import flags
from grr.server.grr_response_server import access_control
from grr.server.grr_response_server import fleetspeak_connector
from grr.server.grr_response_server import server_startup
from grr.server.grr_response_server.bin import api_shell_raw_access_lib
flags.DEFINE_integer(
"page_size", 1000,
"Page size used when paging through collections of items. Default is 1000."
)
flags.DEFINE_string(
"username", None, "Username to use when making raw API calls. If not "
"specified, USER environment variable value will be used.")
flags.DEFINE_string(
"exec_code", None,
"If present, no IPython shell is started but the code given in "
"the flag is run instead (comparable to the -c option of "
"IPython). The code will be able to use a predefined "
"global 'grrapi' object.")
flags.DEFINE_string(
"exec_file", None,
"If present, no IPython shell is started but the code given in "
"command file is supplied as input instead. The code "
"will be able to use a predefined global 'grrapi' "
"object.")
