def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
_ = cmd, args, stdout, stderr, return_val, time_taken, knowledge_base
installed = rdf_client.SoftwarePackage.InstallState.INSTALLED
soft = rdf_client.SoftwarePackage(
name="Package1",
description="Desc1",
version="1",
architecture="amd64",
install_state=installed)
yield soft
soft = rdf_client.SoftwarePackage(
name="Package2",
description="Desc2",
version="1",
architecture="i386",
install_state=installed)
yield soft
# Also yield something random so we can test return type filtering.
yield rdf_client.StatEntry()
# Also yield an anomaly to test that.
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY", symptom="could not parse gremlins.")
def Parse(self, query, result, knowledge_base):
"""Parse the WMI packages output."""
_ = query, knowledge_base
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
soft = rdf_client.SoftwarePackage(name=result["Name"],
description=result["Description"],
version=result["Version"],
install_state=status)
yield soft
def Parse(self, query, result, knowledge_base):
"""Parse the WMI packages output."""
_ = query, knowledge_base
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
result = result.ToDict()
# InstalledOn comes back in a godawful format such as '7/10/2013'.
installed_on = self.AmericanDateToEpoch(result.get("InstalledOn", ""))
soft = rdf_client.SoftwarePackage(
name=result.get("HotFixID"),
description=result.get("Caption"),
installed_by=result.get("InstalledBy"),
install_state=status,
installed_on=installed_on)
yield soft
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the yum output."""
_ = stderr, time_taken, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
for line in stdout.splitlines()[1:]: # Ignore first line
cols = line.split()
name_arch, version, source = cols
name, arch = name_arch.split(".")
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
yield rdf_client.SoftwarePackage(name=name,
publisher=source,
version=version,
architecture=arch,
install_state=status)
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the dpkg output."""
_ = stderr, time_taken, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
column_lengths = []
i = 0
for i, line in enumerate(stdout.splitlines()):
if line.startswith("+++-"):
# This is a special header line that determines column size.
for col in line.split("-")[1:]:
if not re.match("=*", col):
raise parser.ParseError(
"Invalid header parsing for %s at line "
"%s" % (cmd, i))
column_lengths.append(len(col))
break
if column_lengths:
remaining_lines = stdout.splitlines()[i + 1:]
for i, line in enumerate(remaining_lines):
cols = line.split(None, len(column_lengths))
# The status column is ignored in column_lengths.
if len(column_lengths) == 4:
# Installed, Name, Version, Architecture, Description
status, name, version, arch, desc = cols
elif len(column_lengths) == 3:
# Older versions of dpkg don't print Architecture
status, name, version, desc = cols
arch = None
else:
raise ValueError(
"Bad number of columns in dpkg --list output: %s" %
len(column_lengths))
# Status is potentially 3 columns, but always at least two, desired and
# actual state. We only care about actual state.
if status[1] == "i":
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
else:
status = rdf_client.SoftwarePackage.InstallState.UNKNOWN
yield rdf_client.SoftwarePackage(name=name,
description=desc,
version=version,
architecture=arch,
install_state=status)
def Parse(self, stat, file_object, knowledge_base):
"""Parse the status file."""
_, _ = stat, knowledge_base
try:
sw_data = file_object.read()
for pkg in deb822.Packages.iter_paragraphs(sw_data.splitlines()):
if self.installed_re.match(pkg["Status"]):
soft = rdf_client.SoftwarePackage(
name=pkg["Package"],
description=pkg["Description"],
version=pkg["Version"],
architecture=pkg["Architecture"],
publisher=pkg["Maintainer"],
install_state="INSTALLED")
yield soft
except SystemError:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
symptom="Invalid dpkg status file")
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the rpm -qa output."""
_ = time_taken, args, knowledge_base # Unused.
rpm_re = re.compile(r"^(\w[-\w\+]+?)-(\d.*)$")
self.CheckReturn(cmd, return_val)
for line in stdout.splitlines():
pkg_match = rpm_re.match(line.strip())
if pkg_match:
name, version = pkg_match.groups()
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
yield rdf_client.SoftwarePackage(
name=name, version=version, install_state=status)
for line in stderr.splitlines():
if "error: rpmdbNextIterator: skipping h#" in line:
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY", symptom="Broken rpm database.")
break
def testBigQueryPluginFallbackToAFF4(self):
plugin_args = bigquery_plugin.BigQueryOutputPluginArgs()
responses = [
rdf_client.StatEntry(
pathspec=rdf_paths.PathSpec(path="/中国新闻网新闻中", pathtype="OS")),
rdf_client.Process(pid=42),
rdf_client.Process(pid=43),
rdf_client.SoftwarePackage(name="test.deb")
]
plugin = bigquery_plugin.BigQueryOutputPlugin(
source_urn=self.results_urn,
output_base_urn=self.base_urn,
args=plugin_args,
token=self.token)
plugin.InitializeState()
messages = []
for response in responses:
messages.append(
rdf_flows.GrrMessage(source=self.client_id, payload=response))
with test_lib.FakeTime(1445995873):
with mock.patch.object(bigquery,
"GetBigQueryClient") as mock_bigquery:
mock_bigquery.return_value.configure_mock(**{
"InsertData.side_effect":
bigquery.BigQueryJobUploadError()
})
with test_lib.ConfigOverrider(
{"BigQuery.max_upload_failures": 2}):
for message in messages:
plugin.ProcessResponses([message])
plugin.Flush()
# We have 3 output types but a limit of 2 upload failures, so we
# shouldn't try the third one.
self.assertEqual(
mock_bigquery.return_value.InsertData.call_count, 2)
# We should have written a data file and a schema file for each type.
for output_name in [
"ExportedFile", "ExportedProcess",
"AutoExportedSoftwarePackage"
]:
schema_fd = aff4.FACTORY.Open(self.base_urn.Add(
"C-1000000000000000_Results_%s_1445995873.schema" %
output_name),
token=self.token)
data_fd = aff4.FACTORY.Open(self.base_urn.Add(
"C-1000000000000000_Results_%s_1445995873.data" % output_name),
token=self.token)
actual_fd = gzip.GzipFile(None, "r", 9, data_fd)
if output_name == "ExportedFile":
self.CompareSchemaToKnownGood(json.load(schema_fd))
self.assertEqual(
json.load(actual_fd)["urn"],
self.client_id.Add("/fs/os/中国新闻网新闻中"))
elif output_name == "ExportedProcess":
self.assertEqual(json.load(schema_fd)[1]["name"], "pid")
expected_pids = ["42", "43"]
for i, line in enumerate(actual_fd):
self.assertEqual(json.loads(line)["pid"], expected_pids[i])
else:
self.assertEqual(json.load(schema_fd)[1]["name"], "name")
self.assertEqual(json.load(actual_fd)["name"], "test.deb")
# Process the same messages to make sure we're re-using the filehandles.
with test_lib.FakeTime(1445995878):
with mock.patch.object(bigquery,
"GetBigQueryClient") as mock_bigquery:
mock_bigquery.return_value.configure_mock(**{
"InsertData.side_effect":
bigquery.BigQueryJobUploadError()
})
with test_lib.ConfigOverrider(
{"BigQuery.max_upload_failures": 2}):
for message in messages:
plugin.ProcessResponses([message])
plugin.Flush()
# We shouldn't call insertdata at all because we have passed max
# failures already
self.assertEqual(
mock_bigquery.return_value.InsertData.call_count, 0)
expected_line_counts = {
"ExportedFile": 2,
"ExportedProcess": 4,
"AutoExportedSoftwarePackage": 2
}
for output_name in [
"ExportedFile", "ExportedProcess",
"AutoExportedSoftwarePackage"
]:
data_fd = aff4.FACTORY.Open(self.base_urn.Add(
"C-1000000000000000_Results_%s_1445995873.data" % output_name),
token=self.token)
actual_fd = gzip.GzipFile(None, "r", 9, data_fd)
self.assertEqual(sum(1 for line in actual_fd),
expected_line_counts[output_name])