def Notify(self, message_type, subject, msg):
"""Send a notification to the originating user.
Args:
message_type: The type of the message. This allows the UI to format
a link to the original object e.g. "ViewObject" or "HostInformation"
subject: The urn of the AFF4 object of interest in this link.
msg: A free form textual message.
"""
user = self.context.creator
# Don't send notifications to system users.
if (self.runner_args.notify_to_user and
user not in aff4_users.GRRUser.SYSTEM_USERS):
# Prefix the message with the hostname of the client we are running
# against.
if self.runner_args.client_id:
client_fd = aff4.FACTORY.Open(
self.runner_args.client_id, mode="rw", token=self.token)
hostname = client_fd.Get(client_fd.Schema.HOSTNAME) or ""
client_msg = "%s: %s" % (hostname, msg)
else:
client_msg = msg
# Add notification to the User object.
fd = aff4.FACTORY.Create(
aff4.ROOT_URN.Add("users").Add(user),
aff4_users.GRRUser,
mode="rw",
token=self.token)
# Queue notifications to the user.
fd.Notify(message_type, subject, client_msg, self.session_id)
fd.Close()
# Add notifications to the flow.
notification = rdf_flows.Notification(
type=message_type,
subject=utils.SmartUnicode(subject),
message=utils.SmartUnicode(msg),
source=self.session_id,
timestamp=rdfvalue.RDFDatetime.Now())
data_store.DB.Set(self.session_id,
self.flow_obj.Schema.NOTIFICATION,
notification,
replace=False,
sync=False,
token=self.token)
# Disable further notifications.
self.context.user_notified = True
# Allow the flow to either specify an event name or an event handler URN.
notification_event = (self.runner_args.notification_event or
self.runner_args.notification_urn)
if notification_event:
if self.context.state == rdf_flows.FlowContext.State.ERROR:
status = rdf_flows.FlowNotification.Status.ERROR
else:
status = rdf_flows.FlowNotification.Status.OK
event = rdf_flows.FlowNotification(
session_id=self.context.session_id,
flow_name=self.runner_args.flow_name,
client_id=self.runner_args.client_id,
status=status)
self.flow_obj.Publish(notification_event, message=event)
def RenderValue(self, value): """Renders given value into plain old python objects.""" return self._IncludeTypeInfo(utils.SmartUnicode(value), value)
def RenderValue(self, value):
result = {}
for k, v in value.items():
result[utils.SmartUnicode(k)] = self._PassThrough(v)
return self._IncludeTypeInfo(result, value)
def DeleteAttributes(self,
subject,
attributes,
start=None,
end=None,
token=None,
sync=False):
"""Remove all the attributes from this subject."""
_ = sync # Unused attribute, mongo is always synced.
subject = utils.SmartUnicode(subject)
self.security_manager.CheckDataStoreAccess(token, [subject], "w")
if not attributes:
# Nothing to delete.
return
# Build a spec to select the subject and any of the predicates.
spec = {
"$and": [
dict(subject=subject),
{
"$or": [
dict(predicate=utils.SmartUnicode(x))
for x in attributes
]
},
]
}
if not start and not end:
# Just delete all the versions.
self.versioned_collection.remove(spec)
self.latest_collection.remove(spec)
return
unversioned_spec = {
"$and": [
dict(subject=subject),
{
"$or": [
dict(predicate=utils.SmartUnicode(x))
for x in attributes
]
},
]
}
if start:
spec["$and"].append(dict(timestamp={"$gte": start}))
if not end:
# We can optimize this case since the latest version will always
# be unchanged or deleted.
self.versioned_collection.remove(spec)
self.latest_collection.remove(spec)
return
spec["$and"].append(dict(timestamp={"$lte": end}))
self.versioned_collection.remove(spec)
to_delete = set(attributes)
to_set = {}
cursor = self.versioned_collection.find(unversioned_spec).sort(
"timestamp")
for document in cursor:
value = document["value"]
predicate = document["predicate"]
to_delete.discard(predicate)
timestamp = document["timestamp"]
prefix = predicate.split(":", 1)[0]
document = dict(subject=subject,
timestamp=timestamp,
predicate=predicate,
prefix=prefix,
value=value)
to_set[predicate] = document
if to_delete:
delete_spec = {
"$and": [
dict(subject=subject),
{
"$or": [
dict(predicate=utils.SmartUnicode(x))
for x in attributes
]
},
]
}
self.latest_collection.remove(delete_spec)
if to_set:
for document in to_set.itervalues():
self.latest_collection.update(dict(subject=subject,
predicate=predicate,
prefix=prefix),
document,
upsert=True,
w=1 if sync else 0)
def __init__(self,
base_fd,
pathspec=None,
progress_callback=None,
full_pathspec=None):
super(File, self).__init__(base_fd,
pathspec=pathspec,
full_pathspec=full_pathspec,
progress_callback=progress_callback)
if base_fd is None:
self.pathspec.Append(pathspec)
# We can stack on another directory, which means we concatenate their
# directory with ours.
elif base_fd.IsDirectory():
self.pathspec.last.path = utils.JoinPath(self.pathspec.last.path,
pathspec.path)
else:
raise IOError(
"File handler can not be stacked on another handler.")
self.path = self.pathspec.last.path
# We can optionally apply a global offset to the file.
if self.pathspec[0].HasField("offset"):
self.file_offset = self.pathspec[0].offset
self.pathspec.last.path_options = paths.PathSpec.Options.CASE_LITERAL
self.FileHacks()
self.filename = client_utils.CanonicalPathToLocalPath(self.path)
error = None
# Pythonic way - duck typing. Is the handle a directory?
try:
if not self.files:
# Note that the encoding of local path is system specific
local_path = client_utils.CanonicalPathToLocalPath(self.path +
"/")
self.files = [
utils.SmartUnicode(entry)
for entry in os.listdir(local_path)
]
# Some filesystems do not support unicode properly
except UnicodeEncodeError as e:
raise IOError(str(e))
except (IOError, OSError) as e:
self.files = []
error = e
# Ok, it's not. Is it a file then?
try:
with FileHandleManager(self.filename) as fd:
if pathspec.last.HasField("file_size_override"):
self.size = pathspec.last.file_size_override - self.file_offset
else:
# Work out how large the file is.
if self.size is None:
fd.Seek(0, 2)
end = fd.Tell()
if end == 0:
# This file is not seekable, we just use the default.
end = pathspec.last.file_size_override
self.size = end - self.file_offset
error = None
# Some filesystems do not support unicode properly
except UnicodeEncodeError as e:
raise IOError(str(e))
except IOError as e:
if error:
error = e
if error is not None:
raise error # pylint: disable=raising-bad-type
def Generate(self, collection, token=None):
"""Generates archive from a given collection.
Iterates the collection and generates an archive by yielding contents
of every referenced AFF4Stream.
Args:
collection: Iterable with items that point to aff4 paths.
token: User's ACLToken.
Yields:
Binary chunks comprising the generated archive.
"""
hashes = set()
for fd_urn_batch in utils.Grouper(self._ItemsToUrns(collection),
self.BATCH_SIZE):
fds_to_write = {}
for fd in aff4.FACTORY.MultiOpen(fd_urn_batch, token=token):
self.total_files += 1
if not self.predicate(fd):
self.ignored_files.append(utils.SmartUnicode(fd.urn))
continue
# Any file-like object with data in AFF4 should inherit AFF4Stream.
if isinstance(fd, aff4.AFF4Stream):
archive_path = os.path.join(self.prefix, *fd.urn.Split())
sha256_hash = fd.Get(fd.Schema.HASH,
rdf_crypto.Hash()).sha256
if not sha256_hash:
continue
self.archived_files += 1
content_path = os.path.join(self.prefix, "hashes",
str(sha256_hash))
if sha256_hash not in hashes:
# Make sure size of the original file is passed. It's required
# when output_writer is StreamingTarWriter.
st = os.stat_result(
(0644, 0, 0, 0, 0, 0, fd.size, 0, 0, 0))
fds_to_write[fd] = (content_path, st)
hashes.add(sha256_hash)
up_prefix = "../" * len(fd.urn.Split())
yield self.archive_generator.WriteSymlink(
up_prefix + content_path, archive_path)
if fds_to_write:
prev_fd = None
for fd, chunk, exception in aff4.AFF4Stream.MultiStream(
fds_to_write):
if exception:
logging.exception(exception)
self.archived_files -= 1
self.failed_files.append(utils.SmartUnicode(fd.urn))
continue
if prev_fd != fd:
if prev_fd:
yield self.archive_generator.WriteFileFooter()
prev_fd = fd
content_path, st = fds_to_write[fd]
yield self.archive_generator.WriteFileHeader(
content_path, st=st)
yield self.archive_generator.WriteFileChunk(chunk)
if self.archive_generator.is_file_write_in_progress:
yield self.archive_generator.WriteFileFooter()
for chunk in self._WriteDescription():
yield chunk
yield self.archive_generator.Close()
def AllTextsPresent(self, texts):
body = self.driver.find_element_by_tag_name("body").text
for text in texts:
if utils.SmartUnicode(text) not in body:
return False
return True
def ResolveMulti(self,
subject,
attributes,
timestamp=None,
limit=None,
token=None):
subject = utils.SmartUnicode(subject)
self.security_manager.CheckDataStoreAccess(
token, [subject], self.GetRequiredResolveAccess(attributes))
# Does timestamp represent a range?
if isinstance(timestamp, (list, tuple)):
start, end = timestamp # pylint: disable=unpacking-non-sequence
else:
start, end = -1, 1 << 65
start = int(start)
end = int(end)
if isinstance(attributes, str):
attributes = [attributes]
try:
record = self.subjects[subject]
except KeyError:
return
# Holds all the attributes which matched. Keys are attribute names, values
# are lists of timestamped data.
results = {}
for attribute in attributes:
for attr, values in record.iteritems():
if attr == attribute:
for value, ts in values:
results_list = results.setdefault(attribute, [])
# If we are always after the latest ts we clear older ones.
if (results_list and timestamp == self.NEWEST_TIMESTAMP
and results_list[0][1] < ts):
results_list = []
results[attribute] = results_list
# Timestamp outside the range, drop it.
elif ts < start or ts > end:
continue
results_list.append((attribute, ts, value))
# Return the results in the same order they requested.
remaining_limit = limit
for attribute in attributes:
# This returns triples of (attribute_name, timestamp, data). We want to
# sort by timestamp.
for _, ts, data in sorted(results.get(attribute, []),
key=lambda x: x[1],
reverse=True):
if remaining_limit:
remaining_limit -= 1
if remaining_limit == 0:
yield (attribute, data, ts)
return
yield (attribute, data, ts)
def ResolvePrefix(self,
subject,
attribute_prefix,
token=None,
timestamp=None,
limit=None):
"""Resolve all attributes for a subject starting with a prefix."""
subject = utils.SmartUnicode(subject)
self.security_manager.CheckDataStoreAccess(
token, [subject], self.GetRequiredResolveAccess(attribute_prefix))
if timestamp in [None, self.NEWEST_TIMESTAMP, self.ALL_TIMESTAMPS]:
start, end = 0, (2**63) - 1
# Does timestamp represent a range?
elif isinstance(timestamp, (list, tuple)):
start, end = timestamp # pylint: disable=unpacking-non-sequence
else:
raise ValueError("Invalid timestamp: %s" % timestamp)
start = int(start)
end = int(end)
if isinstance(attribute_prefix, str):
attribute_prefix = [attribute_prefix]
subject = utils.SmartUnicode(subject)
try:
record = self.subjects[subject]
except KeyError:
return []
# Holds all the attributes which matched. Keys are attribute names, values
# are lists of timestamped data.
results = {}
nr_results = 0
for prefix in attribute_prefix:
for attribute, values in record.iteritems():
if limit and nr_results >= limit:
break
if utils.SmartStr(attribute).startswith(prefix):
for value, ts in values:
results_list = results.setdefault(attribute, [])
# If we are always after the latest ts we clear older ones.
if (results_list and timestamp == self.NEWEST_TIMESTAMP
and results_list[0][1] < ts):
results_list = []
results[attribute] = results_list
# Timestamp outside the range, drop it.
elif ts < start or ts > end:
continue
results_list.append((attribute, ts, value))
nr_results += 1
if limit and nr_results >= limit:
break
result = []
for attribute_name, values in sorted(results.items()):
# Values are triples of (attribute_name, timestamp, data). We want to
# sort by timestamp.
for _, ts, data in sorted(values, key=lambda x: x[1],
reverse=True):
# Return triples (attribute_name, data, timestamp).
result.append((attribute_name, data, ts))
return result
def Request(self):
"""Create the Approval object and notify the Approval Granter."""
approval_id = "approval:%X" % utils.PRNG.GetULong()
approval_urn = self.BuildApprovalUrn(approval_id)
subject_title = self.BuildSubjectTitle()
email_msg_id = email.utils.make_msgid()
with aff4.FACTORY.Create(
approval_urn, self.approval_type, mode="w",
token=self.token) as approval_request:
approval_request.Set(approval_request.Schema.SUBJECT(self.subject_urn))
approval_request.Set(
approval_request.Schema.REQUESTOR(self.token.username))
approval_request.Set(approval_request.Schema.REASON(self.reason))
approval_request.Set(approval_request.Schema.EMAIL_MSG_ID(email_msg_id))
cc_addresses = (self.email_cc_address,
config.CONFIG.Get("Email.approval_cc_address"))
email_cc = ",".join(filter(None, cc_addresses))
# When we reply with the approval we want to cc all the people to whom the
# original approval was sent, to avoid people approving stuff that was
# already approved.
if email_cc:
reply_cc = ",".join((self.approver, email_cc))
else:
reply_cc = self.approver
approval_request.Set(approval_request.Schema.EMAIL_CC(reply_cc))
approval_request.Set(
approval_request.Schema.NOTIFIED_USERS(self.approver))
# We add ourselves as an approver as well (The requirement is that we have
# 2 approvers, so the requester is automatically an approver).
approval_request.AddAttribute(
approval_request.Schema.APPROVER(self.token.username))
approval_link_urns = self.BuildApprovalSymlinksUrns(approval_id)
for link_urn in approval_link_urns:
with aff4.FACTORY.Create(
link_urn, aff4.AFF4Symlink, mode="w", token=self.token) as link:
link.Set(link.Schema.SYMLINK_TARGET(approval_urn))
# Notify to the users.
for user in self.approver.split(","):
user = user.strip()
try:
fd = aff4.FACTORY.Open(
aff4.ROOT_URN.Add("users").Add(user),
aff4_type=aff4_users.GRRUser,
mode="rw",
token=self.token)
except aff4.InstantiationError:
continue
fd.Notify("GrantAccess", approval_urn,
"Please grant access to %s" % subject_title, "")
fd.Close()
if not config.CONFIG.Get("Email.send_approval_emails"):
return approval_urn
reason = self.CreateReasonHTML(utils.SmartStr(self.reason))
subject_template = jinja2.Template(
"Approval for {{ user }} to access {{ client }}.", autoescape=True)
subject = subject_template.render(
user=utils.SmartUnicode(self.token.username),
client=utils.SmartUnicode(subject_title))
template = jinja2.Template(
"""
<html><body><h1>Approval to access
<a href='{{ admin_ui }}/#/{{ approval_url }}'>{{ subject_title }}</a>
requested.</h1>
The user "{{ username }}" has requested access to
<a href='{{ admin_ui }}/#/{{ approval_url }}'>{{ subject_title }}</a>
for the purpose of "{{ reason }}".
Please click <a href='{{ admin_ui }}/#/{{ approval_url }}'>
here
</a> to review this request and then grant access.
<p>Thanks,</p>
<p>{{ signature }}</p>
<p>{{ image|safe }}</p>
</body></html>""",
autoescape=True)
body = template.render(
username=utils.SmartUnicode(self.token.username),
reason=utils.SmartUnicode(reason),
admin_ui=utils.SmartUnicode(config.CONFIG["AdminUI.url"]),
subject_title=utils.SmartUnicode(subject_title),
approval_url=utils.SmartUnicode(
self.BuildApprovalReviewUrlPath(approval_id)),
# If you feel like it, add a funny cat picture here :)
image=utils.SmartUnicode(config.CONFIG["Email.approval_signature"]),
signature=utils.SmartUnicode(config.CONFIG["Email.signature"]))
email_alerts.EMAIL_ALERTER.SendEmail(
self.approver,
utils.SmartStr(self.token.username),
utils.SmartStr(subject),
utils.SmartStr(body),
is_html=True,
cc_addresses=email_cc,
message_id=email_msg_id)
return approval_urn
def Grant(self):
"""Create the Approval object and notify the Approval Granter."""
approvals_root_urn = aff4.ROOT_URN.Add("ACL").Add(
self.subject_urn.Path()).Add(self.delegate)
children_urns = list(
aff4.FACTORY.ListChildren(approvals_root_urn, token=self.token))
if not children_urns:
raise access_control.UnauthorizedAccess(
"No approval found for user %s" % utils.SmartStr(self.token.username),
subject=self.subject_urn)
approvals = aff4.FACTORY.MultiOpen(
children_urns, mode="r", aff4_type=Approval, token=self.token)
found_approval_urn = None
for approval in approvals:
approval_reason = approval.Get(approval.Schema.REASON)
if (utils.SmartUnicode(approval_reason) == utils.SmartUnicode(
self.reason) and (not found_approval_urn or
approval_reason.age > found_approval_urn.age)):
found_approval_urn = approval.urn
found_approval_urn.age = approval_reason.age
if not found_approval_urn:
raise access_control.UnauthorizedAccess(
"No approval with reason '%s' found for user %s" %
(utils.SmartStr(self.reason), utils.SmartStr(self.token.username)),
subject=self.subject_urn)
subject_title = self.BuildSubjectTitle()
access_url = self.BuildAccessUrl()
# This object must already exist.
try:
approval_request = aff4.FACTORY.Open(
found_approval_urn,
mode="rw",
aff4_type=self.approval_type,
token=self.token)
except IOError:
raise access_control.UnauthorizedAccess(
"Approval object does not exist.", requested_access="rw")
# We are now an approver for this request.
approval_request.AddAttribute(
approval_request.Schema.APPROVER(self.token.username))
email_msg_id = utils.SmartStr(
approval_request.Get(approval_request.Schema.EMAIL_MSG_ID))
email_cc = utils.SmartStr(
approval_request.Get(approval_request.Schema.EMAIL_CC))
approval_request.Close(sync=True)
# Notify to the user.
fd = aff4.FACTORY.Create(
aff4.ROOT_URN.Add("users").Add(self.delegate),
aff4_users.GRRUser,
mode="rw",
token=self.token)
fd.Notify("ViewObject", self.subject_urn,
"%s has granted you access to %s." % (self.token.username,
subject_title), "")
fd.Close()
if not config.CONFIG.Get("Email.send_approval_emails"):
return found_approval_urn
reason = self.CreateReasonHTML(utils.SmartStr(self.reason))
subject_template = jinja2.Template(
"Approval for {{ user }} to access {{ client }}.", autoescape=True)
subject = subject_template.render(
user=utils.SmartUnicode(self.delegate),
client=utils.SmartUnicode(subject_title))
template = jinja2.Template(
"""
<html><body><h1>Access to
<a href='{{ admin_ui }}/#/{{ subject_url }}'>{{ subject_title }}</a>
granted.</h1>
The user {{ username }} has granted access to
<a href='{{ admin_ui }}/#/{{ subject_url }}'>{{ subject_title }}</a> for the
purpose of "%(reason)s".
Please click <a href='{{ admin_ui }}/#/{{ subject_url }}'>here</a> to access it.
<p>Thanks,</p>
<p>{{ signature }}</p>
</body></html>""",
autoescape=True)
body = template.render(
subject_title=utils.SmartUnicode(subject_title),
username=utils.SmartUnicode(self.token.username),
reason=utils.SmartUnicode(reason),
admin_ui=utils.SmartUnicode(config.CONFIG["AdminUI.url"]),
subject_url=utils.SmartUnicode(access_url),
signature=utils.SmartUnicode(config.CONFIG["Email.signature"]))
# Email subject should match approval request, and we add message id
# references so they are grouped together in a thread by gmail.
headers = {"In-Reply-To": email_msg_id, "References": email_msg_id}
email_alerts.EMAIL_ALERTER.SendEmail(
utils.SmartStr(self.delegate),
utils.SmartStr(self.token.username),
utils.SmartStr(subject),
utils.SmartStr(body),
is_html=True,
cc_addresses=email_cc,
headers=headers)
return found_approval_urn
def __init__(self, request=None, responses=None, auth_required=True):
self.status = None # A GrrStatus rdfvalue object.
self.success = True
self.request = request
self._auth_required = auth_required
if request:
self.request_data = rdf_protodict.Dict(request.data)
self._responses = []
self._dropped_responses = []
if responses:
# This may not be needed if we can assume that responses are
# returned in lexical order from the data_store.
responses.sort(key=operator.attrgetter("response_id"))
# The iterator that was returned as part of these responses. This should
# be passed back to actions that expect an iterator.
self.iterator = None
# Filter the responses by authorized states
for msg in responses:
# Check if the message is authenticated correctly.
if msg.auth_state == msg.AuthorizationState.DESYNCHRONIZED or (
self._auth_required and
msg.auth_state != msg.AuthorizationState.AUTHENTICATED):
logging.warning("%s: Messages must be authenticated (Auth state %s)",
msg.session_id, msg.auth_state)
self._dropped_responses.append(msg)
# Skip this message - it is invalid
continue
# Check for iterators
if msg.type == msg.Type.ITERATOR:
self.iterator = rdf_client.Iterator(msg.payload)
continue
# Look for a status message
if msg.type == msg.Type.STATUS:
# Our status is set to the first status message that we see in
# the responses. We ignore all other messages after that.
self.status = rdf_flows.GrrStatus(msg.payload)
# Check this to see if the call succeeded
self.success = self.status.status == self.status.ReturnedStatus.OK
# Ignore all other messages
break
# Use this message
self._responses.append(msg)
if self.status is None:
# This is a special case of de-synchronized messages.
if self._dropped_responses:
logging.error("De-synchronized messages detected:\n" + "\n".join(
[utils.SmartUnicode(x) for x in self._dropped_responses]))
if responses:
self._LogFlowState(responses)
raise FlowError("No valid Status message.")
# This is the raw message accessible while going through the iterator
self.message = None
def CallFlow(self,
flow_name=None,
next_state=None,
sync=True,
request_data=None,
client_id=None,
base_session_id=None,
**kwargs):
"""Creates a new flow and send its responses to a state.
This creates a new flow. The flow may send back many responses which will be
queued by the framework until the flow terminates. The final status message
will cause the entire transaction to be committed to the specified state.
Args:
flow_name: The name of the flow to invoke.
next_state: The state in this flow, that responses to this
message should go to.
sync: If True start the flow inline on the calling thread, else schedule
a worker to actually start the child flow.
request_data: Any dict provided here will be available in the
RequestState protobuf. The Responses object maintains a reference
to this protobuf for use in the execution of the state method. (so
you can access this data by responses.request). There is no
format mandated on this data but it may be a serialized protobuf.
client_id: If given, the flow is started for this client.
base_session_id: A URN which will be used to build a URN.
**kwargs: Arguments for the child flow.
Raises:
FlowRunnerError: If next_state is not one of the allowed next states.
Returns:
The URN of the child flow which was created.
"""
client_id = client_id or self.runner_args.client_id
# This looks very much like CallClient() above - we prepare a request state,
# and add it to our queue - any responses from the child flow will return to
# the request state and the stated next_state. Note however, that there is
# no client_id or actual request message here because we directly invoke the
# child flow rather than queue anything for it.
state = rdf_flows.RequestState(
id=self.GetNextOutboundId(),
session_id=utils.SmartUnicode(self.session_id),
client_id=client_id,
next_state=next_state,
response_count=0)
if request_data:
state.data = rdf_protodict.Dict().FromDict(request_data)
# If the urn is passed explicitly (e.g. from the hunt runner) use that,
# otherwise use the urn from the flow_runner args. If both are None, create
# a new collection and give the urn to the flow object.
logs_urn = self._GetLogsCollectionURN(
kwargs.pop("logs_collection_urn", None) or
self.runner_args.logs_collection_urn)
# If we were called with write_intermediate_results, propagate down to
# child flows. This allows write_intermediate_results to be set to True
# either at the top level parent, or somewhere in the middle of
# the call chain.
write_intermediate = (kwargs.pop("write_intermediate_results", False) or
self.runner_args.write_intermediate_results)
try:
event_id = self.runner_args.event_id
except AttributeError:
event_id = None
# Create the new child flow but do not notify the user about it.
child_urn = self.flow_obj.StartFlow(
client_id=client_id,
flow_name=flow_name,
base_session_id=base_session_id or self.session_id,
event_id=event_id,
request_state=state,
token=self.token,
notify_to_user=False,
parent_flow=self.flow_obj,
sync=sync,
queue=self.runner_args.queue,
write_intermediate_results=write_intermediate,
logs_collection_urn=logs_urn,
creator=self.context.creator,
**kwargs)
self.QueueRequest(state)
return child_urn
def CallClient(self,
action_cls,
request=None,
next_state=None,
client_id=None,
request_data=None,
start_time=None,
**kwargs):
"""Calls the client asynchronously.
This sends a message to the client to invoke an Action. The run
action may send back many responses. These will be queued by the
framework until a status message is sent by the client. The status
message will cause the entire transaction to be committed to the
specified state.
Args:
action_cls: The function to call on the client.
request: The request to send to the client. If not specified (Or None) we
create a new RDFValue using the kwargs.
next_state: The state in this flow, that responses to this
message should go to.
client_id: rdf_client.ClientURN to send the request to.
request_data: A dict which will be available in the RequestState
protobuf. The Responses object maintains a reference to this
protobuf for use in the execution of the state method. (so you can
access this data by responses.request). Valid values are
strings, unicode and protobufs.
start_time: Call the client at this time. This Delays the client request
for into the future.
**kwargs: These args will be used to construct the client action semantic
protobuf.
Raises:
FlowRunnerError: If next_state is not one of the allowed next states.
RuntimeError: The request passed to the client does not have the correct
type.
"""
if client_id is None:
client_id = self.runner_args.client_id
if client_id is None:
raise FlowRunnerError("CallClient() is used on a flow which was not "
"started with a client.")
if not isinstance(client_id, rdf_client.ClientURN):
# Try turning it into a ClientURN
client_id = rdf_client.ClientURN(client_id)
if action_cls.in_rdfvalue is None:
if request:
raise RuntimeError("Client action %s does not expect args." %
action_cls.__name__)
else:
if request is None:
# Create a new rdf request.
request = action_cls.in_rdfvalue(**kwargs)
else:
# Verify that the request type matches the client action requirements.
if not isinstance(request, action_cls.in_rdfvalue):
raise RuntimeError("Client action expected %s but got %s" %
(action_cls.in_rdfvalue, type(request)))
outbound_id = self.GetNextOutboundId()
# Create a new request state
state = rdf_flows.RequestState(
id=outbound_id,
session_id=self.session_id,
next_state=next_state,
client_id=client_id)
if request_data is not None:
state.data = rdf_protodict.Dict(request_data)
# Send the message with the request state
msg = rdf_flows.GrrMessage(
session_id=utils.SmartUnicode(self.session_id),
name=action_cls.__name__,
request_id=outbound_id,
priority=self.runner_args.priority,
require_fastpoll=self.runner_args.require_fastpoll,
queue=client_id.Queue(),
payload=request,
generate_task_id=True)
if self.context.remaining_cpu_quota:
msg.cpu_limit = int(self.context.remaining_cpu_quota)
cpu_usage = self.context.client_resources.cpu_usage
if self.runner_args.cpu_limit:
msg.cpu_limit = max(self.runner_args.cpu_limit - cpu_usage.user_cpu_time -
cpu_usage.system_cpu_time, 0)
if msg.cpu_limit == 0:
raise FlowRunnerError("CPU limit exceeded.")
if self.runner_args.network_bytes_limit:
msg.network_bytes_limit = max(self.runner_args.network_bytes_limit -
self.context.network_bytes_sent, 0)
if msg.network_bytes_limit == 0:
raise FlowRunnerError("Network limit exceeded.")
state.request = msg
self.QueueRequest(state, timestamp=start_time)
def __unicode__(self): return utils.SmartUnicode(self._string_urn)
def testRecentReasonBox(self):
test_reason = u"ástæða"
self.Open("/")
with self.ACLChecksDisabled():
token = access_control.ACLToken(username="******",
reason=test_reason)
self.GrantClientApproval("C.0000000000000006", token=token)
self.Type("client_query", "C.0000000000000006")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000006", self.GetText,
"css=span[type=subject]")
# Choose client 6
self.Click("css=td:contains('0006')")
self.WaitUntil(self.IsTextPresent, u"Access reason: %s" % test_reason)
# By now we should have a recent reason set, let's see if it shows up in the
# ACL dialog.
self.Type("client_query", "C.0000000000000001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1
self.Click("css=td:contains('0001')")
# This should be rejected now and a form request is made.
self.WaitUntil(self.IsElementPresent,
"css=h3:contains('Create a new approval')")
options = self.GetText("css=select[id=acl_recent_reasons]").split("\n")
self.assertEqual(len(options), 2)
self.assertEqual(options[0].strip(), "Enter New Reason...")
self.assertEqual(options[1].strip(), test_reason)
# The reason text box should be there and enabled.
element = self.GetElement("css=input[id=acl_reason]")
self.assertTrue(element.is_enabled())
self.Select("css=select[id=acl_recent_reasons]", test_reason)
# Make sure clicking the recent reason greys out the reason text box.
element = self.GetElement("css=input[id=acl_reason]")
self.assertFalse(element.is_enabled())
# Ok now submit this.
self.Type("css=input[id=acl_approver]", "test")
self.ClickUntilNotVisible("acl_dialog_submit")
# And make sure the approval was created...
fd = aff4.FACTORY.Open("aff4:/ACL/C.0000000000000001/test",
token=self.token)
approvals = list(fd.ListChildren())
self.assertEqual(len(approvals), 1)
# ... using the correct reason.
self.assertEqual(
utils.SmartUnicode(approvals[0].Basename().decode("base64")),
test_reason)
def ProcessMessage(self, message=None, event=None):
"""Processes this event."""
_ = event
nanny_msg = ""
crash_details = message.payload
client_id = crash_details.client_id
# The session id of the flow that crashed.
session_id = crash_details.session_id
flow_obj = aff4.FACTORY.Open(session_id, token=self.token)
# Log.
logging.info("Client crash reported, client %s.", client_id)
# Only kill the flow if it does not handle its own crashes. Some flows
# restart the client and therefore expect to get a crash notification.
if flow_obj.handles_crashes:
return
# Export.
stats.STATS.IncrementCounter("grr_client_crashes")
# Write crash data to AFF4.
if data_store.RelationalDBReadEnabled():
client = data_store.REL_DB.ReadClient(client_id.Basename())
client_info = client.startup_info.client_info
else:
client = aff4.FACTORY.Open(client_id, token=self.token)
client_info = client.Get(client.Schema.CLIENT_INFO)
crash_details.client_info = client_info
crash_details.crash_type = self.well_known_session_id
self.WriteAllCrashDetails(client_id,
crash_details,
flow_session_id=session_id)
# Also send email.
to_send = []
try:
hunt_session_id = self._ExtractHuntId(session_id)
if hunt_session_id and hunt_session_id != session_id:
hunt_obj = aff4.FACTORY.Open(hunt_session_id,
aff4_type=implementation.GRRHunt,
token=self.token)
email = hunt_obj.runner_args.crash_alert_email
if email:
to_send.append(email)
except aff4.InstantiationError:
logging.error("Failed to open hunt %s.", hunt_session_id)
email = config.CONFIG["Monitoring.alert_email"]
if email:
to_send.append(email)
for email_address in to_send:
if crash_details.nanny_status:
nanny_msg = "Nanny status: %s" % crash_details.nanny_status
client = aff4.FACTORY.Open(client_id, token=self.token)
hostname = client.Get(client.Schema.HOSTNAME)
url = "/clients/%s" % client_id.Basename()
body = self.__class__.mail_template.render(
client_id=client_id,
admin_ui=config.CONFIG["AdminUI.url"],
hostname=utils.SmartUnicode(hostname),
context=utils.SmartUnicode(flow_obj.context),
state=utils.SmartUnicode(flow_obj.state),
args=utils.SmartUnicode(flow_obj.args),
runner_args=utils.SmartUnicode(flow_obj.runner_args),
urn=url,
nanny_msg=utils.SmartUnicode(nanny_msg),
signature=config.CONFIG["Email.signature"]),
email_alerts.EMAIL_ALERTER.SendEmail(
email_address,
"GRR server",
"Client %s reported a crash." % client_id,
utils.SmartStr(body),
is_html=True)
if nanny_msg:
msg = "Client crashed, " + nanny_msg
else:
msg = "Client crashed."
# Now terminate the flow.
flow.GRRFlow.TerminateFlow(session_id,
reason=msg,
token=self.token,
force=True)
def testRecentReasonBox(self):
test_reason = u"ástæða"
self.Open("/")
with self.ACLChecksDisabled():
token = access_control.ACLToken(username="******", reason=test_reason)
self.RequestAndGrantClientApproval("C.0000000000000006", token=token)
self.Type("client_query", "C.0000000000000006")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000006", self.GetText,
"css=span[type=subject]")
# Choose client 6
self.Click("css=td:contains('0006')")
self.WaitUntil(self.IsTextPresent, u"Access reason: %s" % test_reason)
# By now we should have a recent reason set, let's see if it shows up in the
# ACL dialog.
self.Type("client_query", "C.0000000000000001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1
self.Click("css=td:contains('0001')")
# We do not have an approval, so check that the hint is shown, that the
# interrogate button is disabled and that the menu is disabled.
self.WaitUntil(self.IsElementPresent, "css=div.no-approval")
self.WaitUntil(self.IsElementPresent,
"css=button:contains('Interrogate')[disabled]")
self.WaitUntil(self.IsElementPresent, "css=a.nav-link.disabled")
# Request an approval.
self.Click("css=button[name=requestApproval]")
self.WaitUntil(self.IsElementPresent,
"css=h3:contains('Create a new approval')")
self.WaitUntilEqual(2, self.GetCssCount, "css=grr-request-approval-dialog "
"select[name=acl_recent_reasons] option")
self.assertEqual("Enter New Reason...", self.GetText(
"css=grr-request-approval-dialog "
"select[name=acl_recent_reasons] option:nth(0)"))
self.assertEqual(test_reason, self.GetText(
"css=grr-request-approval-dialog "
"select[name=acl_recent_reasons] option:nth(1)"))
# The reason text box should be there and enabled.
element = self.GetElement(
"css=grr-request-approval-dialog input[name=acl_reason]")
self.assertTrue(element.is_enabled())
self.Select(
"css=grr-request-approval-dialog select[name=acl_recent_reasons]",
test_reason)
# Make sure clicking the recent reason greys out the reason text box.
element = self.GetElement(
"css=grr-request-approval-dialog input[name=acl_reason]")
self.assertFalse(element.is_enabled())
# Ok now submit this.
self.Type("css=grr-request-approval-dialog input[name=acl_approver]",
"test")
self.Click(
"css=grr-request-approval-dialog button[name=Proceed]:not([disabled])")
# "Request Approval" dialog should go away.
self.WaitUntilNot(self.IsVisible, "css=.modal-backdrop")
# And make sure the approval was created...
def GetApprovals():
fd = aff4.FACTORY.Open(
"aff4:/ACL/C.0000000000000001/test", token=self.token)
return list(fd.ListChildren())
self.WaitUntilEqual(1, lambda: len(GetApprovals()))
# ... using the correct reason.
approvals = GetApprovals()
approval = aff4.FACTORY.Open(approvals[0], token=self.token)
self.assertEqual(
utils.SmartUnicode(approval.Get(approval.Schema.REASON)), test_reason)
def Render(self, unused_args, token=None):
"""Build the data structure representing the config."""
sections = {}
for descriptor in config_lib.CONFIG.type_infos:
if descriptor.section in sections:
continue
is_bad_section = self._IsBadSection(descriptor.section)
section_data = {}
for parameter in self._ListParametersInSection(descriptor.section):
try:
if parameter in self.redacted_options or is_bad_section:
is_default = False
parameter_data = {"type": "redacted"}
else:
option_value = config_lib.CONFIG.Get(parameter,
default=None)
raw_value = config_lib.CONFIG.GetRaw(parameter,
default=None)
is_default = option_value is None or raw_value is None
if is_default:
option_value = config_lib.CONFIG.Get(parameter)
raw_value = config_lib.CONFIG.GetRaw(parameter)
value_type = "plain"
if isinstance(option_value, rdfvalue.RDFValue):
option_value = api_value_renderers.RenderValue(
option_value)
else:
if isinstance(option_value, str):
raw_value = option_value = None
value_type = "binary"
else:
option_value = utils.SmartUnicode(option_value)
interpolated_value = config_lib.CONFIG.InterpolateValue(
raw_value)
is_expanded = option_value != interpolated_value
parameter_data = {
"raw_value": raw_value,
"option_value": option_value,
"is_expanded": is_expanded,
"is_default": is_default,
"type": value_type
}
except (config_lib.Error, type_info.TypeValueError) as e:
parameter_data = {"type": "error", "error_message": str(e)}
section_data[parameter] = parameter_data
sections[descriptor.section] = section_data
return sections
def RegisterClientError(self, client_id, log_message=None, backtrace=None):
error = rdfvalue.HuntError(client_id=client_id, backtrace=backtrace)
if log_message:
error.log_message = utils.SmartUnicode(log_message)
self._AddObjectToCollection(error, self.clients_errors_collection_urn)
def MultiSet(self,
subject,
values,
timestamp=None,
token=None,
replace=True,
sync=True,
to_delete=None):
"""Set multiple predicates' values for this subject in one operation."""
self.security_manager.CheckDataStoreAccess(token, [subject], "w")
if timestamp is None:
timestamp = time.time() * 1e6
# Prepare a mongo bulk insert for all the values.
documents = []
subject = utils.SmartUnicode(subject)
to_delete = set(to_delete or [])
latest = {}
# Build a document for each unique timestamp.
for attribute, sequence in values.items():
for value in sequence:
if isinstance(value, tuple):
value, entry_timestamp = value
else:
entry_timestamp = timestamp
if entry_timestamp is None:
entry_timestamp = timestamp
predicate = utils.SmartUnicode(attribute)
prefix = predicate.split(":", 1)[0]
document = dict(subject=subject,
timestamp=int(entry_timestamp),
predicate=predicate,
prefix=prefix)
_Encode(document, value)
documents.append(document)
latest[predicate] = document
# Replacing means to delete all versions of the attribute first.
if replace:
to_delete.add(attribute)
if to_delete:
self.DeleteAttributes(subject, to_delete, token=token)
# Just write using bulk insert mode.
if documents:
try:
self.versioned_collection.insert(documents, w=1 if sync else 0)
except errors.PyMongoError as e:
logging.error("Mongo Error %s", e)
raise data_store.Error(utils.SmartUnicode(e))
# Maintain the latest documents in the latest collection.
for predicate, document in latest.items():
document.pop("_id", None)
self.latest_collection.update(dict(subject=subject,
predicate=predicate,
prefix=prefix),
document,
upsert=True,
w=1 if sync else 0)
def __unicode__(self): return utils.SmartUnicode(self._value)
def Check(self, method, url, payload=None, replace=None):
"""Records output of a given url accessed with a given method.
Args:
method: HTTP method. May be "GET" or "POST".
url: String repesenting an url.
payload: JSON-able payload that will be sent when "POST" method is used.
replace: Dictionary of key->value pairs. In the recorded JSON output
every "key" string will be replaced with its "value"
counterpart. This way we can properly handle dynamically
generated values (like hunts IDs) in the regression data.
Raises:
ValueError: if unsupported method argument is passed. Currently only
"GET", "POST", "DELETE" and "PATCH" are supported.
RuntimeError: if request was handled by an unexpected API method (
every test is annotated with an "api_method" attribute
that points to the expected API method).
"""
if self.use_api_v2:
url = url.replace("/api/", "/api/v2/")
parsed_url = urlparse.urlparse(url)
request = utils.DataObject(method=method,
scheme="http",
path=parsed_url.path,
environ={
"SERVER_NAME": "foo.bar",
"SERVER_PORT": 1234
},
user="******",
body="")
request.META = {"CONTENT_TYPE": "application/json"}
if method == "GET":
request.GET = dict(urlparse.parse_qsl(parsed_url.query))
elif method in ["POST", "DELETE", "PATCH"]:
# NOTE: this is a temporary trick. Payloads in regression tests
# are using the API v1 (non-proto3) format. Here we're reparsing
# them and serializing as proto3 JSON.
# TODO(user): Make regression tests payload format-agnostic.
# I.e. use protobuf and API client library to send requests.
if self.use_api_v2 and payload:
router_matcher = http_api.RouterMatcher()
_, metadata, _ = router_matcher.MatchRouter(request)
rdf_args = metadata.args_type()
rdf_args.FromDict(payload)
proto_args = metadata.args_type.protobuf()
proto_args.ParseFromString(rdf_args.SerializeToString())
request.body = json_format.MessageToJson(proto_args)
payload = json.loads(request.body)
else:
request.body = json.dumps(payload or "")
else:
raise ValueError("Unsupported method: %s." % method)
with self.NoAuthorizationChecks():
http_response = http_api.RenderHttpResponse(request)
api_method = http_response["X-API-Method"]
if api_method != self.__class__.api_method:
raise RuntimeError("Request was handled by an unexpected method. "
"Expected %s, got %s." %
(self.__class__.api_method, api_method))
if hasattr(http_response, "streaming_content"):
# We don't know the nature of response content, but we force it to be
# unicode. It's a strategy that's good enough for testing purposes.
content = utils.SmartUnicode("".join(
http_response.streaming_content))
else:
content = http_response.content
xssi_token = ")]}'\n"
if content.startswith(xssi_token):
content = content[len(xssi_token):]
# replace the values of all tracebacks by <traceback content>
regex = re.compile(r'"traceBack": "Traceback[^"\\]*(?:\\.[^"\\]*)*"',
re.DOTALL)
content = regex.sub('"traceBack": "<traceback content>"', content)
if replace:
if hasattr(replace, "__call__"):
replace = replace()
# We reverse sort replacements by length to avoid cases when
# replacements include each other and therefore order
# of replacements affects the result.
for substr in sorted(replace, key=len, reverse=True):
repl = replace[substr]
if hasattr(substr, "sub"): # regex
content = substr.sub(repl, content)
url = substr.sub(repl, url)
else:
content = content.replace(substr, repl)
url = url.replace(substr, repl)
# We treat streaming content purely as strings and don't expect it to
# contain JSON data.
if hasattr(http_response, "streaming_content"):
parsed_content = content
else:
parsed_content = json.loads(content)
check_result = dict(api_method=api_method,
method=method,
url=url,
test_class=self.__class__.__name__,
response=parsed_content)
if payload:
check_result["request_payload"] = payload
# Type stripping only makes sense for version 1 of the API.
if not self.use_api_v2:
stripped_content = api_value_renderers.StripTypeInfo(
parsed_content)
if parsed_content != stripped_content:
check_result["type_stripped_response"] = stripped_content
self.checks.append(check_result)
def SerializeToDataStore(self): return utils.SmartUnicode(self._value)
def Run(self, unused_arg):
# psutil will cause an active loop on Windows 2000
if platform.system() == "Windows" and platform.version().startswith(
"5.0"):
raise RuntimeError("ListProcesses not supported on Windows 2000")
for proc in psutil.process_iter():
response = rdfvalue.Process()
for field in [
"pid", "ppid", "name", "exe", "username", "terminal"
]:
try:
if not hasattr(proc, field) or not getattr(proc, field):
continue
value = getattr(proc, field)
if isinstance(value, (int, long)):
setattr(response, field, value)
else:
setattr(response, field, utils.SmartUnicode(value))
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
for arg in proc.cmdline:
response.cmdline.append(utils.SmartUnicode(arg))
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
response.nice = proc.get_nice()
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
# Not available on Windows.
if hasattr(proc, "uids"):
(response.real_uid, response.effective_uid,
response.saved_uid) = proc.uids
(response.real_gid, response.effective_gid,
response.saved_gid) = proc.gids
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
response.ctime = long(proc.create_time * 1e6)
response.status = str(proc.status)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
# Not available on OSX.
if hasattr(proc, "getcwd"):
response.cwd = utils.SmartUnicode(proc.getcwd())
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
response.num_threads = proc.get_num_threads()
except (psutil.NoSuchProcess, psutil.AccessDenied, RuntimeError):
pass
try:
(response.user_cpu_time,
response.system_cpu_time) = proc.get_cpu_times()
# This is very time consuming so we do not collect cpu_percent here.
# response.cpu_percent = proc.get_cpu_percent()
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
try:
response.RSS_size, response.VMS_size = proc.get_memory_info()
response.memory_percent = proc.get_memory_percent()
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
# Due to a bug in psutil, this function is disabled for now.
# try:
# for f in proc.get_open_files():
# response.open_files.append(utils.SmartUnicode(f.path))
# except (psutil.NoSuchProcess, psutil.AccessDenied):
# pass
try:
for c in proc.get_connections():
conn = response.connections.Append(family=c.family,
type=c.type,
pid=proc.pid)
try:
conn.state = c.status
except ValueError:
logging.info(
"Encountered unknown connection status (%s).",
c.status)
try:
conn.local_address.ip, conn.local_address.port = c.laddr
# Could be in state LISTEN.
if c.raddr:
conn.remote_address.ip, conn.remote_address.port = c.raddr
except AttributeError:
conn.local_address.ip, conn.local_address.port = c.local_address
# Could be in state LISTEN.
if c.remote_address:
(conn.remote_address.ip,
conn.remote_address.port) = c.remote_address
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
self.SendReply(response)
# Reading information here is slow so we heartbeat between processes.
self.Progress()
def __unicode__(self): return utils.SmartUnicode(str(self))
def RenderValue(self, value): return self._IncludeTypeInfo(utils.SmartUnicode(value), value)
def SerializeToDataStore(self): return utils.SmartUnicode(self._string_urn)
def RenderValue(self, value): result = utils.SmartUnicode(value.Uncompress()) return self._IncludeTypeInfo(result, value)
def SetStatus(self, status, message="", backtrace=None):
"""Set a status to report back to the server."""
self.status.status = status
self.status.error_message = utils.SmartUnicode(message)
if backtrace:
self.status.backtrace = utils.SmartUnicode(backtrace)
