def WaitForClientToEnroll(admin_ui_port):
"""Waits for an already started client to enroll.
If the client doesn't enroll within ~100 seconds, main process gets killed.
Args:
admin_ui_port: AdminUI port to be used with API client library to check for
an enrolled client.
Returns:
A string with an enrolled client's client id.
Raises:
ClientEnrollmentTimeout: if the client fails to enroll in time.
"""
api_endpoint = "http://localhost:%d" % admin_ui_port
start_time = time.time()
while time.time() - start_time < _CLIENT_ENROLLMENT_WAIT_TIMEOUT_SECS * 10:
try:
api_client = api.InitHttp(api_endpoint=api_endpoint)
clients = list(api_client.SearchClients(query="."))
except requests.exceptions.ConnectionError:
# print("Connection error (%s), waiting..." % api_endpoint)
time.sleep(_CLIENT_ENROLLMENT_CHECK_INTERVAL)
continue
if clients:
return clients[0].client_id
print("No clients enrolled, waiting...")
time.sleep(_CLIENT_ENROLLMENT_CHECK_INTERVAL)
raise ClientEnrollmentTimeout("Client didn't enroll.")
def SetUp(self,
reason,
grr_server_url,
grr_username,
grr_password,
approvers=None,
verify=True):
"""Initializes a GRR hunt result collector.
Args:
reason: justification for GRR access.
grr_server_url: GRR server URL.
grr_username: GRR username.
grr_password: GRR password.
approvers: list of GRR approval recipients.
verify: boolean, whether to verify the GRR server's x509 certificate.
"""
grr_auth = (grr_username, grr_password)
self.approvers = []
if approvers:
self.approvers = [
item.strip() for item in approvers.strip().split(',')
]
self.grr_api = grr_api.InitHttp(api_endpoint=grr_server_url,
auth=grr_auth,
verify=verify)
self.output_path = tempfile.mkdtemp()
self.reason = reason
def SetUp(self,
reason,
grr_server_url,
grr_username,
grr_password,
approvers=None,
verify=True):
"""Initializes a GRR hunt result collector.
Args:
reason (str): justification for GRR access.
grr_server_url (str): GRR server URL.
grr_username (str): GRR username.
grr_password (str): GRR password.
approvers (Optional[str]): comma-separated GRR approval recipients.
verify (Optional[bool]): True to indicate GRR server's x509 certificate
should be verified.
"""
grr_auth = (grr_username, grr_password)
self.approvers = []
if approvers:
self.approvers = [item.strip() for item in approvers.split(',')]
self.grr_api = grr_api.InitHttp(api_endpoint=grr_server_url,
auth=grr_auth,
verify=verify)
self.output_path = tempfile.mkdtemp()
self.reason = reason
def Initialize(self):
"""Initializes state in preparation for running end-to-end tests.
Only needs to be called once.
"""
appveyor_root_url = os.environ.get(self.APPVEYOR_API_VARNAME, None)
if appveyor_root_url:
logging.info("Using Appveyor API at %s", appveyor_root_url)
# See https://www.appveyor.com/docs/build-worker-api/
self._appveyor_tests_endpoint = urlparse.urljoin(
appveyor_root_url, "api/tests")
self._appveyor_messages_endpoint = urlparse.urljoin(
appveyor_root_url, "api/build/messages")
logging.info("Connecting to GRR API at %s", self._api_endpoint)
password = self._api_password
if not password:
password = getpass.getpass(
prompt="Please enter the API password for "
"user '%s': " % self._api_user)
self._grr_api = api.InitHttp(api_endpoint=self._api_endpoint,
auth=(self._api_user, password))
# Make sure binaries required by tests are uploaded to the datastore.
if self._upload_test_binaries:
binary_paths = self._GetUploadedBinaries()
if self.LINUX_TEST_BINARY_PATH not in binary_paths:
self._UploadBinary(self.LINUX_TEST_BINARY_NAME,
self.LINUX_TEST_BINARY_PATH)
if self.WINDOWS_TEST_BINARY_PATH not in binary_paths:
self._UploadBinary(self.WINDOWS_TEST_BINARY_NAME,
self.WINDOWS_TEST_BINARY_PATH)
def testConnectionFails(self):
client_id = self.SetupClient(0)
api = grr_api.InitHttp(api_endpoint=self.__class__.ssl_endpoint,
trust_env=False)
with self.assertRaises(requests.exceptions.SSLError):
api.Client(client_id=client_id).Get()
def testInitHttpApiFailsForOutdatedVersion(self):
handle = self._Handle()
with mock.patch.object(metadata.ApiGetGrrVersionHandler, "Handle",
handle):
with self.assertRaises(errors.VersionMismatchError):
grr_api.InitHttp(api_endpoint=self.endpoint,
validate_version=True)
def testInitHttpApiSucceedsForOutdatedVersionWithDisableValidation(self):
handle = self._Handle()
with mock.patch.object(metadata.ApiGetGrrVersionHandler, "Handle",
handle):
# Should not raise.
grr_api.InitHttp(api_endpoint=self.endpoint,
validate_version=False)
def main(argv=None):
if not argv:
argv = sys.argv[1:]
arg_parser = GrrApiShellArgParser()
flags = arg_parser.parse_args(args=argv)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stderr))
if flags.debug:
logging.getLogger().setLevel(logging.DEBUG)
auth = None
if flags.basic_auth_username:
auth = (flags.basic_auth_username, flags.basic_auth_password or "")
verify = True
if flags.no_check_certificate:
verify = False
grrapi = api.InitHttp(api_endpoint=flags.api_endpoint,
page_size=flags.page_size,
auth=auth,
verify=verify)
if flags.exec_code and flags.exec_file:
print("--exec_code --exec_file flags can't be supplied together")
sys.exit(1)
elif flags.exec_code:
# pylint: disable=exec-used
exec(flags.exec_code, dict(grrapi=grrapi))
# pylint: enable=exec-used
elif flags.exec_file:
api_shell_lib.ExecFile(flags.exec_file, grrapi)
else:
api_shell_lib.IPShell([sys.argv[0]], user_ns=dict(grrapi=grrapi))
def __init__(
self,
hostname,
reason,
grr_server_url,
grr_auth,
approvers=None,
verbose=False,
keepalive=False):
"""Initializes a GRR collector.
Args:
hostname: hostname of machine.
reason: justification for GRR access.
grr_server_url: GRR server URL.
grr_auth: Tuple containing a (username, password) combination.
approvers: comma-separated list of GRR approval recipients.
verbose: toggle for verbose output.
keepalive: toggle for scheduling a KeepAlive flow.
"""
super(GRRHostCollector, self).__init__(verbose=verbose)
if approvers is None:
approvers = []
self.output_path = tempfile.mkdtemp()
self.grr_api = grr_api.InitHttp(api_endpoint=grr_server_url, auth=grr_auth)
self.host = hostname
self.reason = reason
self.approvers = approvers
self._client_id = self._GetClientId(hostname)
self._client = None
self.keepalive = keepalive
self.flow_id = None
def KillClient(admin_ui_port, client_id):
"""Kills a given client."""
api_endpoint = "http://localhost:%d" % admin_ui_port
api_client = api.InitHttp(api_endpoint=api_endpoint)
f = api_client.Client(client_id).CreateFlow("Kill")
f.WaitUntilDone(timeout=60)
def testConnectionFails(self):
client_id = self.SetupClient(0)
# TODO: Enable version validation.
api = grr_api.InitHttp(
api_endpoint=self.__class__.ssl_endpoint, validate_version=False)
with self.assertRaises(requests.exceptions.SSLError):
api.Client(client_id=client_id).Get()
def setUp(self):
super(ApiE2ETest, self).setUp()
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.port = ApiE2ETest.server_port
self.endpoint = "http://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
def __init__(self):
Analyzer.__init__(self)
self.grr_url = self.get_param('config.url', None,
'Missing GRR API endpoint')
self.grr_user = self.get_param('config.username', None,
'Missing GRR username')
self.grr_passwd = self.get_param('config.password', None,
'Missing GRR password')
self.proxies = self.get_param('config.proxy', None)
self.grrapi = api.InitHttp(api_endpoint=self.grr_url,
auth=(self.grr_user, self.grr_passwd))
def testProxyConnection(self):
client_urn = self.SetupClient(0)
api = grr_api.InitHttp(
api_endpoint=self.endpoint,
proxies={"https": "localhost:%d" % self.proxy_port})
with self.assertRaises(requests.exceptions.ConnectionError):
api.Client(client_id=client_urn.Basename()).Get()
# CONNECT request should point to GRR SSL server.
self.assertEqual(Proxy.requests,
["CONNECT localhost:%d HTTP/1.0" % self.port])
def setUp(self):
super(ApiSSLE2ETest, self).setUp()
key = rdf_crypto.RSAPrivateKey.GenerateKey()
key_path = os.path.join(self.temp_dir, "key.pem")
with open(key_path, "wb") as f:
f.write(key.AsPEM())
subject = issuer = x509.Name([
x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"),
])
cert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(
key.GetPublicKey().GetRawPublicKey()).serial_number(
x509.random_serial_number()).not_valid_before(
datetime.datetime.utcnow()).not_valid_after(
datetime.datetime.utcnow() +
datetime.timedelta(days=1)).add_extension(
x509.SubjectAlternativeName(
[x509.DNSName(u"localhost")]),
critical=False,
).sign(key.GetRawPrivateKey(), hashes.SHA256(),
backends.default_backend())
cert_path = os.path.join(self.temp_dir, "certificate.pem")
with open(cert_path, "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
self.config_overrider = test_lib.ConfigOverrider({
"AdminUI.enable_ssl":
True,
"AdminUI.ssl_key_file":
key_path,
"AdminUI.ssl_cert_file":
cert_path,
})
self.config_overrider.Start()
self.prev_environ = dict(os.environ)
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
self.port = portpicker.PickUnusedPort()
self.thread = wsgiapp_testlib.ServerThread(self.port)
self.thread.StartAndWaitUntilServing()
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.endpoint = "https://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
def RunEndToEndTests():
"""Runs end-to-end tests against clients using the GRR API."""
ValidateAllTests()
logging.info("Connecting to API at %s", flags.FLAGS.api_endpoint)
password = flags.FLAGS.api_password
if not password:
password = getpass.getpass(prompt="Please enter the API password for "
"user '%s': " % flags.FLAGS.api_user)
grr_api = api.InitHttp(api_endpoint=flags.FLAGS.api_endpoint,
auth=(flags.FLAGS.api_user, password))
logging.info("Fetching client data from the API.")
target_clients = test_base.GetClientTestTargets(
grr_api=grr_api,
client_ids=flags.FLAGS.client_ids,
hostnames=flags.FLAGS.hostnames)
if not target_clients:
raise RuntimeError(
"No clients to test on. Either pass --client_ids or --hostnames "
"or check that corresponding clients checked in recently.")
# Make sure binaries required by tests are uploaded to the datastore.
if flags.FLAGS.upload_test_binaries:
api_response = grr_api._context.SendRequest("ListGrrBinaries", None)
server_paths = {item.path for item in api_response.items}
UploadBinaryIfAbsent(server_paths, "hello", "linux/test/hello")
UploadBinaryIfAbsent(server_paths, "hello.exe",
"windows/test/hello.exe")
results_by_client = {}
max_test_name_len = 0
logging.info("Running tests against %d clients...", len(target_clients))
for client in target_clients:
results_by_client[client.client_id] = RunTestsAgainstClient(
grr_api, client)
for test_name in results_by_client[client.client_id]:
max_test_name_len = max(max_test_name_len, len(test_name))
for client_urn, results in results_by_client.iteritems():
logging.info("Results for %s:", client_urn)
for test_name, result in sorted(results.items()):
res = "[ OK ]"
if result.errors or result.failures:
res = "[ FAIL ]"
# Print a summary line for the test, using left-alignment for the test
# name and right alignment for the result.
logging.info("\t%s %s",
(test_name + ":").ljust(max_test_name_len + 1),
res.rjust(10))
def setUp(self):
super().setUp()
prev_environ = dict(os.environ)
def _CleanUpEnviron():
os.environ.clear()
os.environ.update(prev_environ)
self.addCleanup(_CleanUpEnviron)
os.environ["REQUESTS_CA_BUNDLE"] = self.__class__.ssl_cert_path
self.api = grr_api.InitHttp(api_endpoint=self.__class__.ssl_endpoint)
def setUp(self):
super(ApiSslWithConfigurationInEnvVarsE2ETest, self).setUp()
prev_environ = dict(os.environ)
def _CleanUpEnviron():
os.environ.clear()
os.environ.update(prev_environ)
self.addCleanup(_CleanUpEnviron)
os.environ["REQUESTS_CA_BUNDLE"] = self.cert_path
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
def connect(self, params):
self.logger.info("Connect: Connecting...")
self.api_endpoint = params.get('api_endpoint')
self.username = params.get('credentials').get('username')
self.password = params.get('credentials').get('password')
ssl_verify = params.get('ssl_verify')
try:
self.grrapi = api.InitHttp(api_endpoint=self.api_endpoint,
auth=(self.username, self.password),
verify=ssl_verify)
except Exception as e:
self.logger.error(
"Please provide valid options to connect to the GRR API endpoint"
)
raise e
def testProxyConnection(self):
client_id = self.SetupClient(0)
# TODO: Enable version validation.
api = grr_api.InitHttp(
api_endpoint=self.__class__.ssl_endpoint,
proxies={"https": "localhost:%d" % self.proxy_port},
validate_version=False)
with self.assertRaises(requests.exceptions.ConnectionError):
api.Client(client_id=client_id).Get()
# CONNECT request should point to GRR SSL server.
self.assertEqual(
Proxy.requests,
["CONNECT localhost:%d HTTP/1.0" % self.__class__.ssl_port])
def setUp(self):
super(ApiE2ETest, self).setUp()
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.port = ApiE2ETest.server_port
self.endpoint = "http://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
self.poll_stubber = utils.MultiStubber(
(grr_api_utils, "DEFAULT_POLL_INTERVAL", 0.1),
(grr_api_utils, "DEFAULT_POLL_TIMEOUT", 10))
self.poll_stubber.Start()
def get():
"""Lazily returns the GRR API object."""
global _API
if _API is None:
if not FLAGS.grr_http_api_endpoint:
raise ValueError("HTTP API endpoint has not been specified.")
if not FLAGS.grr_auth_api_user:
raise ValueError("API user name has not been specified.")
if not FLAGS.grr_auth_password:
raise ValueError("API user password has not been specified.")
auth = (FLAGS.grr_auth_api_user, FLAGS.grr_auth_password)
_API = api.InitHttp(api_endpoint=FLAGS.grr_http_api_endpoint,
auth=auth)
return _API
def WaitForAPIEndpoint(port):
"""Waits for API endpoint to come online."""
api_endpoint = "http://localhost:%d" % port
start_time = time.time()
while time.time() - start_time < _WAIT_TIMEOUT_SECS:
try:
grrapi = api.InitHttp(api_endpoint=api_endpoint)
grrapi.ListGrrBinaries()
return grrapi
except (requests.exceptions.ConnectionError, ConnectionRefusedError):
print("Connection error (%s), waiting..." % api_endpoint)
time.sleep(_CHECK_INTERVAL)
continue
raise APIEndpointTimeoutError("API endpoint %s didn't come up." %
api_endpoint)
def setUpClass(cls) -> None:
"""Performs all initialization needed to interface with GRR's API."""
# This is a mixin class intended to be used with `absltest.TestCase`.
super(ColabTestMixin, cls).setUpClass() # pytype: disable=attribute-error
# TODO(hanuszczak): `TestInit` is awful, does a lot of unnecessary stuff and
# should be avoided. However, because of all the global state that GRR has
# currently, it is extremely hard figure out which parts need initialization
# and which do not. Once AFF4 is gone, hopefully this should become much
# simpler and `TestInit` will no longer be necessary.
testing_startup.TestInit()
port = portpicker.pick_unused_port()
cls._server_thread = wsgiapp_testlib.ServerThread(port, name="ServerThread")
cls._server_thread.StartAndWaitUntilServing()
_api._API = api.InitHttp(api_endpoint="http://localhost:{}".format(port)) # pylint: disable=protected-access
def RunEndToEndTests():
"""Runs end-to-end tests against clients using the GRR API."""
ValidateAllTests()
logging.info("Connecting to API at %s", flags.FLAGS.api_endpoint)
password = flags.FLAGS.api_password
if not password:
password = getpass.getpass(prompt="Please enter the API password for "
"user '%s': " % flags.FLAGS.api_user)
grr_api = api.InitHttp(api_endpoint=flags.FLAGS.api_endpoint,
auth=(flags.FLAGS.api_user, password))
logging.info("Fetching client data from the API.")
target_clients = test_base.GetClientTestTargets(
grr_api=grr_api,
client_ids=flags.FLAGS.client_ids,
hostnames=flags.FLAGS.hostnames)
if not target_clients:
raise RuntimeError(
"No clients to test on. Either pass --client_ids or --hostnames "
"or check that corresponding clients checked in recently.")
results_by_client = {}
max_test_name_len = 0
logging.info("Running tests against %d clients...", len(target_clients))
for client in target_clients:
results_by_client[client.client_id] = RunTestsAgainstClient(
grr_api, client)
for test_name in results_by_client[client.client_id]:
max_test_name_len = max(max_test_name_len, len(test_name))
for client_urn, results in results_by_client.iteritems():
logging.info("Results for %s:", client_urn)
for test_name, result in sorted(results.items()):
res = "[ OK ]"
if result.errors or result.failures:
res = "[ FAIL ]"
# Print a summary line for the test, using left-alignment for the test
# name and right alignment for the result.
logging.info("\t%s %s",
(test_name + ":").ljust(max_test_name_len + 1),
res.rjust(10))
def setup(self, reason, grr_server_url, grr_auth, approvers=None):
"""Initializes a GRR hunt result collector.
Args:
reason: justification for GRR access.
grr_server_url: GRR server URL.
grr_auth: Tuple containing a (username, password) combination.
approvers: list of GRR approval recipients.
"""
self.approvers = []
if approvers:
self.approvers = [
item.strip() for item in approvers.strip().split(',')
]
self.grr_api = grr_api.InitHttp(api_endpoint=grr_server_url,
auth=grr_auth)
self.output_path = tempfile.mkdtemp()
self.reason = reason
def createGRRFlow(esid, flow_name):
search = getHits(esid)
grr_url = parser.get('grr', 'grr_url')
grr_user = parser.get('grr', 'grr_user')
grr_pass = parser.get('grr', 'grr_pass')
grrapi = api.InitHttp(api_endpoint=grr_url,
auth=(grr_user, grr_pass))
for result in search['hits']['hits']:
result = result['_source']
message = result['message']
description = str(message)
info = description
if 'source_ip' in result:
source_ip = result['source_ip']
if 'destination_ip' in result:
destination_ip = result['destination_ip']
for ip in source_ip, destination_ip:
search_result = grrapi.SearchClients(ip)
grr_result = {}
client_id = ''
for client in search_result:
# Get client id
client_id = client.client_id
client_last_seen_at = client.data.last_seen_at
grr_result[client_id] = client_last_seen_at
#flow_name = "ListProcesses"
if client_id is None:
pass
# Run flow
flow_obj = grrapi.Client(client_id)
flow_obj.CreateFlow(name=flow_name)
if client_id:
# Redirect to GRR instance
return redirect(grr_url + '/#/clients/' + client_id + '/flows')
else:
return "No matches found for source or destination ip"
def __init__(self, server_link=None, user=None, password=None):
"""
gets default ip address
"""
if server_link is None:
server_link = "http://{}:{}".format(get_ip_address(),
self.default_admin_port)
print(server_link)
if server_link is None or user is None or password is None:
raise ValueError(
"server_link, user and password needs to be provided")
self.server_address = server_link
self.server_user = user
self.server_password = password
self.grr_api = api.InitHttp(api_endpoint=self.server_address,
auth=(self.server_user,
self.server_password))
def __init__(
self, reason, grr_server_url, grr_auth, approvers=None, verbose=False):
"""Initializes a GRR hunt results collector.
Args:
reason: justification for GRR access.
grr_server_url: GRR server URL.
grr_auth: Tuple containing a (username, password) combination.
approvers: list of GRR approval recipients.
verbose: toggle for verbose output.
"""
super(GRRHuntCollector, self).__init__(verbose=verbose)
self._hunt = None
if approvers is None:
approvers = []
self.approvers = approvers
self.grr_api = grr_api.InitHttp(api_endpoint=grr_server_url, auth=grr_auth)
self.hunt_id = None
self.output_path = tempfile.mkdtemp()
self.reason = reason
def setUp(self):
super(ApiIntegrationTest, self).setUp()
api_auth_manager.InitializeApiAuthManager()
self.token.username = "******"
try:
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
except AttributeError:
# Only the NullWebAuthManager supports SetUserName
pass
self.CreateUser(self.token.username)
self.port = ApiIntegrationTest.server_port
self.endpoint = "http://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
poll_stubber = utils.MultiStubber(
(grr_api_utils, "DEFAULT_POLL_INTERVAL", 0.1),
(grr_api_utils, "DEFAULT_POLL_TIMEOUT", 10))
poll_stubber.Start()
self.addCleanup(poll_stubber.Stop)
