def MultiWritePathHistory(self, client_path_histories):
"""Writes a collection of hash and stat entries observed for given paths."""
for client_path, client_path_history in iteritems(
client_path_histories):
if client_path.client_id not in self.metadatas:
raise db.UnknownClientError(client_path.client_id)
path_info = rdf_objects.PathInfo(path_type=client_path.path_type,
components=client_path.components)
for timestamp, stat_entry in iteritems(
client_path_history.stat_entries):
path_record = self._GetPathRecord(client_path.client_id,
path_info,
set_default=False)
if path_record is None:
# TODO(hanuszczak): Provide more details about paths that caused that.
raise db.AtLeastOneUnknownPathError([])
path_record.AddStatEntry(stat_entry, timestamp)
for timestamp, hash_entry in iteritems(
client_path_history.hash_entries):
path_record = self._GetPathRecord(client_path.client_id,
path_info,
set_default=False)
if path_record is None:
# TODO(hanuszczak): Provide more details about paths that caused that.
raise db.AtLeastOneUnknownPathError([])
path_record.AddHashEntry(hash_entry, timestamp)
def WriteClientCrashInfo(self, client_id, crash_info, cursor=None):
"""Writes a new client crash record."""
cursor.execute("SET @now = NOW(6)")
params = {
"client_id": db_utils.ClientIDToInt(client_id),
"crash_info": crash_info.SerializeToBytes(),
}
try:
cursor.execute(
"""
INSERT INTO client_crash_history (client_id, timestamp, crash_info)
VALUES (%(client_id)s, @now, %(crash_info)s)
""", params)
cursor.execute(
"""
UPDATE clients
SET last_crash_timestamp = @now
WHERE client_id = %(client_id)s
""", params)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def DeleteClient(self, client_id):
"""Deletes a client with all associated metadata."""
try:
del self.metadatas[client_id]
except KeyError:
raise db.UnknownClientError(client_id)
self.clients.pop(client_id, None)
self.labels.pop(client_id, None)
self.startup_history.pop(client_id, None)
self.crash_history.pop(client_id, None)
self.client_stats.pop(client_id, None)
for key in [k for k in self.flows if k[0] == client_id]:
self.flows.pop(key)
for key in [k for k in self.flow_requests if k[0] == client_id]:
self.flow_requests.pop(key)
for key in [
k for k in self.flow_processing_requests if k[0] == client_id
]:
self.flow_processing_requests.pop(key)
for key in [
k for k in self.client_action_requests if k[0] == client_id
]:
self.client_action_requests.pop(key)
for kw in self.keywords:
self.keywords[kw].pop(client_id, None)
def WriteClientStats(self,
client_id: Text,
stats: rdf_client_stats.ClientStats,
cursor=None) -> None:
"""Stores a ClientStats instance."""
if stats.timestamp is None:
stats.timestamp = rdfvalue.RDFDatetime.Now()
try:
cursor.execute(
"""
INSERT INTO client_stats (client_id, payload, timestamp)
VALUES (%s, %s, FROM_UNIXTIME(%s))
ON DUPLICATE KEY UPDATE payload=VALUES(payload)
""", [
db_utils.ClientIDToInt(client_id),
stats.SerializeToBytes(),
mysql_utils.RDFDatetimeToTimestamp(stats.timestamp)
])
except MySQLdb.IntegrityError as e:
if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2:
raise db.UnknownClientError(client_id, cause=e)
else:
raise
def WriteClientStats(self, client_id,
stats):
"""Stores a ClientStats instance."""
if client_id not in collection.Flatten(self.ReadAllClientIDs()):
raise db.UnknownClientError(client_id)
self.client_stats[client_id][rdfvalue.RDFDatetime.Now()] = stats
def WriteClientSnapshotHistory(self, clients, cursor=None):
"""Writes the full history for a particular client."""
client_id = clients[0].client_id
latest_timestamp = max(client.timestamp for client in clients)
base_params = {
"client_id": db_utils.ClientIDToInt(client_id),
"latest_timestamp": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp)
}
try:
for client in clients:
startup_info = client.startup_info
client.startup_info = None
params = base_params.copy()
params.update({
"timestamp": mysql_utils.RDFDatetimeToTimestamp(client.timestamp),
"client_snapshot": client.SerializeToString(),
"startup_info": startup_info.SerializeToString(),
})
cursor.execute(
"""
INSERT INTO client_snapshot_history (client_id, timestamp,
client_snapshot)
VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp)s),
%(client_snapshot)s)
""", params)
cursor.execute(
"""
INSERT INTO client_startup_history (client_id, timestamp,
startup_info)
VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp)s),
%(startup_info)s)
""", params)
client.startup_info = startup_info
cursor.execute(
"""
UPDATE clients
SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
WHERE client_id = %(client_id)s
AND (last_snapshot_timestamp IS NULL OR
last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s))
""", base_params)
cursor.execute(
"""
UPDATE clients
SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
WHERE client_id = %(client_id)s
AND (last_startup_timestamp IS NULL OR
last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s))
""", base_params)
except MySQLdb.IntegrityError as error:
raise db.UnknownClientError(client_id, cause=error)
def WriteFlowObject(self, flow_obj):
"""Writes a flow object to the database."""
if flow_obj.client_id not in self.metadatas:
raise db.UnknownClientError(flow_obj.client_id)
clone = flow_obj.Copy()
clone.last_update_time = rdfvalue.RDFDatetime.Now()
self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone
def AddClientKeywords(self, client_id, keywords):
"""Associates the provided keywords with the client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
for kw in keywords:
self.keywords.setdefault(kw, {})
self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()
def AddClientLabels(self, client_id, owner, labels):
"""Attaches a user label to a client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())
for l in labels:
labelset.add(utils.SmartUnicode(l))
def WriteClientStartupInfo(self, client_id, startup_info):
"""Writes a new client startup record."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
ts = rdfvalue.RDFDatetime.Now()
self.metadatas[client_id]["startup_info_timestamp"] = ts
history = self.startup_history.setdefault(client_id, {})
history[ts] = startup_info.SerializeToBytes()
def WriteClientCrashInfo(self, client_id, crash_info):
"""Writes a new client crash record."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
ts = rdfvalue.RDFDatetime.Now()
self.metadatas[client_id]["last_crash_timestamp"] = ts
history = self.crash_history.setdefault(client_id, {})
history[ts] = crash_info.SerializeToBytes()
def WriteClientSnapshotHistory(self, clients, cursor=None):
"""Writes the full history for a particular client."""
client_id = clients[0].client_id
latest_timestamp = max(client.timestamp for client in clients)
query = ""
params = {
"client_id": db_utils.ClientIDToInt(client_id),
"latest_timestamp": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp)
}
for idx, client in enumerate(clients):
startup_info = client.startup_info
client.startup_info = None
query += """
INSERT INTO client_snapshot_history (client_id, timestamp,
client_snapshot)
VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),
%(client_snapshot_{idx})s);
INSERT INTO client_startup_history (client_id, timestamp,
startup_info)
VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),
%(startup_info_{idx})s);
""".format(idx=idx)
params.update({
"timestamp_{idx}".format(idx=idx):
mysql_utils.RDFDatetimeToTimestamp(client.timestamp),
"client_snapshot_{idx}".format(idx=idx):
client.SerializeToString(),
"startup_info_{idx}".format(idx=idx):
startup_info.SerializeToString(),
})
client.startup_info = startup_info
query += """
UPDATE clients
SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
WHERE client_id = %(client_id)s
AND (last_snapshot_timestamp IS NULL OR
last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));
UPDATE clients
SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
WHERE client_id = %(client_id)s
AND (last_startup_timestamp IS NULL OR
last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));
"""
try:
cursor.execute(query, params)
except MySQLdb.IntegrityError as error:
raise db.UnknownClientError(client_id, cause=error)
def WriteClientStats(self, client_id, stats):
"""Stores a ClientStats instance."""
if client_id not in collection.Flatten(self.ReadAllClientIDs()):
raise db.UnknownClientError(client_id)
if stats.timestamp is None:
stats.timestamp = rdfvalue.RDFDatetime.Now()
copy = rdf_client_stats.ClientStats(stats)
self.client_stats[client_id][copy.timestamp] = copy
def WriteClientSnapshot(self, snapshot, cursor=None):
"""Write new client snapshot."""
insert_history_query = (
"INSERT INTO client_snapshot_history(client_id, timestamp, "
"client_snapshot) VALUES (%s, FROM_UNIXTIME(%s), %s)")
insert_startup_query = (
"INSERT INTO client_startup_history(client_id, timestamp, "
"startup_info) VALUES(%s, FROM_UNIXTIME(%s), %s)")
now = rdfvalue.RDFDatetime.Now()
current_timestamp = mysql_utils.RDFDatetimeToTimestamp(now)
client_info = {
"last_snapshot_timestamp": current_timestamp,
"last_startup_timestamp": current_timestamp,
"last_version_string": snapshot.GetGRRVersionString(),
"last_platform": snapshot.knowledge_base.os,
"last_platform_release": snapshot.Uname(),
}
update_clauses = [
"last_snapshot_timestamp = FROM_UNIXTIME(%(last_snapshot_timestamp)s)",
"last_startup_timestamp = FROM_UNIXTIME(%(last_startup_timestamp)s)",
"last_version_string = %(last_version_string)s",
"last_platform = %(last_platform)s",
"last_platform_release = %(last_platform_release)s",
]
update_query = (
"UPDATE clients SET {} WHERE client_id = %(client_id)s".format(
", ".join(update_clauses)))
int_client_id = db_utils.ClientIDToInt(snapshot.client_id)
client_info["client_id"] = int_client_id
startup_info = snapshot.startup_info
snapshot.startup_info = None
try:
cursor.execute(insert_history_query,
(int_client_id, current_timestamp,
snapshot.SerializeToBytes()))
cursor.execute(insert_startup_query,
(int_client_id, current_timestamp,
startup_info.SerializeToBytes()))
cursor.execute(update_query, client_info)
except MySQLdb.IntegrityError as e:
if e.args and e.args[
0] == mysql_error_constants.NO_REFERENCED_ROW_2:
raise db.UnknownClientError(snapshot.client_id, cause=e)
elif e.args and e.args[0] == mysql_error_constants.DUP_ENTRY:
raise mysql_utils.RetryableError(cause=e)
else:
raise
finally:
snapshot.startup_info = startup_info
def _WritePathInfo(self, client_id, path_info):
"""Writes a single path info record for given client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
path_record = self._GetPathRecord(client_id, path_info)
path_record.AddPathInfo(path_info)
parent_path_info = path_info.GetParent()
if parent_path_info is not None:
parent_path_record = self._GetPathRecord(client_id, parent_path_info)
parent_path_record.AddChild(path_info)
def WriteScheduledFlow(
self, scheduled_flow: rdf_flow_objects.ScheduledFlow) -> None:
"""See base class."""
if scheduled_flow.client_id not in self.metadatas:
raise db.UnknownClientError(scheduled_flow.client_id)
if scheduled_flow.creator not in self.users:
raise db.UnknownGRRUserError(scheduled_flow.creator)
full_id = (scheduled_flow.client_id, scheduled_flow.creator,
scheduled_flow.scheduled_flow_id)
self.scheduled_flows[full_id] = scheduled_flow.Copy()
def WriteFlowObject(self, flow_obj, allow_update=True):
"""Writes a flow object to the database."""
if flow_obj.client_id not in self.metadatas:
raise db.UnknownClientError(flow_obj.client_id)
key = (flow_obj.client_id, flow_obj.flow_id)
if not allow_update and key in self.flows:
raise db.FlowExistsError(flow_obj.client_id, flow_obj.flow_id)
clone = flow_obj.Copy()
clone.last_update_time = rdfvalue.RDFDatetime.Now()
self.flows[key] = clone
def AddClientLabels(self, client_id, owner, labels, cursor=None):
"""Attaches a list of user labels to a client."""
cid = db_utils.ClientIDToInt(client_id)
labels = set(labels)
args = [(cid, mysql_utils.Hash(owner), owner, label) for label in labels]
args = list(collection.Flatten(args))
query = """
INSERT IGNORE INTO client_labels
(client_id, owner_username_hash, owner_username, label)
VALUES {}
""".format(", ".join(["(%s, %s, %s, %s)"] * len(labels)))
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshotHistory(self, clients):
"""Writes the full history for a particular client."""
if clients[0].client_id not in self.metadatas:
raise db.UnknownClientError(clients[0].client_id)
for client in clients:
startup_info = client.startup_info
client.startup_info = None
snapshots = self.clients.setdefault(client.client_id, {})
snapshots[client.timestamp] = client.SerializeToString()
startup_infos = self.startup_history.setdefault(client.client_id, {})
startup_infos[client.timestamp] = startup_info.SerializeToString()
client.startup_info = startup_info
def AddClientKeywords(self, client_id, keywords, cursor=None):
"""Associates the provided keywords with the client."""
cid = db_utils.ClientIDToInt(client_id)
keywords = set(keywords)
args = [(cid, mysql_utils.Hash(kw), kw) for kw in keywords]
args = list(collection.Flatten(args))
query = """
INSERT INTO client_keywords (client_id, keyword_hash, keyword)
VALUES {}
ON DUPLICATE KEY UPDATE timestamp = NOW(6)
""".format(", ".join(["(%s, %s, %s)"] * len(keywords)))
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshot(self, snapshot):
"""Writes new client snapshot."""
client_id = snapshot.client_id
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
startup_info = snapshot.startup_info
snapshot.startup_info = None
ts = rdfvalue.RDFDatetime.Now()
history = self.clients.setdefault(client_id, {})
history[ts] = snapshot.SerializeToBytes()
history = self.startup_history.setdefault(client_id, {})
history[ts] = startup_info.SerializeToBytes()
snapshot.startup_info = startup_info
def DeleteClient(self, client_id, cursor=None):
"""Deletes a client with all associated metadata."""
cursor.execute("SELECT COUNT(*) FROM clients WHERE client_id = %s",
[db_utils.ClientIDToInt(client_id)])
if cursor.fetchone()[0] == 0:
raise db.UnknownClientError(client_id)
# Clean out foreign keys first.
cursor.execute(
"""
UPDATE clients SET
last_crash_timestamp = NULL,
last_snapshot_timestamp = NULL,
last_startup_timestamp = NULL
WHERE client_id = %s""", [db_utils.ClientIDToInt(client_id)])
cursor.execute("DELETE FROM clients WHERE client_id = %s",
[db_utils.ClientIDToInt(client_id)])
def WriteClientStartupInfo(self, client_id, startup_info, cursor=None):
"""Writes a new client startup record."""
query = """
SET @now = NOW(6);
INSERT INTO client_startup_history (client_id, timestamp, startup_info)
VALUES (%(client_id)s, @now, %(startup_info)s);
UPDATE clients
SET last_startup_timestamp = @now
WHERE client_id = %(client_id)s;
"""
params = {
"client_id": db_utils.ClientIDToInt(client_id),
"startup_info": startup_info.SerializeToString(),
}
try:
cursor.execute(query, params)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def WritePathInfos(self, client_id, path_infos):
"""Writes a collection of path_info records for a client."""
try:
self._MultiWritePathInfos({client_id: path_infos})
except MySQLdb.IntegrityError as error:
raise db.UnknownClientError(client_id=client_id, cause=error)
def WritePathInfos(
self,
client_id: str,
path_infos: Sequence[rdf_objects.PathInfo],
cursor: Optional[MySQLdb.cursors.Cursor] = None,
) -> None:
"""Writes a collection of path_info records for a client."""
now = rdfvalue.RDFDatetime.Now()
int_client_id = db_utils.ClientIDToInt(client_id)
path_info_values = []
parent_path_info_values = []
stat_entry_keys = []
stat_entry_values = []
hash_entry_keys = []
hash_entry_values = []
for path_info in path_infos:
path = mysql_utils.ComponentsToPath(path_info.components)
key = (
int_client_id,
int(path_info.path_type),
path_info.GetPathID().AsBytes(),
)
path_info_values.append(key +
(mysql_utils.RDFDatetimeToTimestamp(now),
path, bool(path_info.directory),
len(path_info.components)))
if path_info.HasField("stat_entry"):
stat_entry_keys.extend(key)
stat_entry_values.append(
key + (mysql_utils.RDFDatetimeToTimestamp(now),
path_info.stat_entry.SerializeToBytes()))
if path_info.HasField("hash_entry"):
hash_entry_keys.extend(key)
hash_entry_values.append(
key + (mysql_utils.RDFDatetimeToTimestamp(now),
path_info.hash_entry.SerializeToBytes(),
path_info.hash_entry.sha256.AsBytes()))
# TODO(hanuszczak): Implement a trie in order to avoid inserting
# duplicated records.
for parent_path_info in path_info.GetAncestors():
path = mysql_utils.ComponentsToPath(
parent_path_info.components)
parent_path_info_values.append((
int_client_id,
int(parent_path_info.path_type),
parent_path_info.GetPathID().AsBytes(),
path,
len(parent_path_info.components),
))
if path_info_values:
query = """
INSERT INTO client_paths(client_id, path_type, path_id,
timestamp,
path, directory, depth)
VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s, %s)
ON DUPLICATE KEY UPDATE
timestamp = VALUES(timestamp),
directory = directory OR VALUES(directory)
"""
try:
cursor.executemany(query, path_info_values)
except MySQLdb.IntegrityError as error:
raise db.UnknownClientError(client_id=client_id, cause=error)
if parent_path_info_values:
query = """
INSERT INTO client_paths(client_id, path_type, path_id, path,
directory, depth)
VALUES (%s, %s, %s, %s, TRUE, %s)
ON DUPLICATE KEY UPDATE
directory = TRUE,
timestamp = NOW(6)
"""
cursor.executemany(query, parent_path_info_values)
if stat_entry_values:
query = """
INSERT INTO client_path_stat_entries(client_id, path_type, path_id,
timestamp,
stat_entry)
VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s)
"""
cursor.executemany(query, stat_entry_values)
condition = "(client_id = %s AND path_type = %s AND path_id = %s)"
query = """
UPDATE client_paths
FORCE INDEX (PRIMARY)
SET last_stat_entry_timestamp = FROM_UNIXTIME(%s)
WHERE {}
""".format(" OR ".join([condition] * len(stat_entry_values)))
params = [mysql_utils.RDFDatetimeToTimestamp(now)
] + stat_entry_keys
cursor.execute(query, params)
if hash_entry_values:
query = """
INSERT INTO client_path_hash_entries(client_id, path_type, path_id,
timestamp,
hash_entry, sha256)
VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s)
"""
cursor.executemany(query, hash_entry_values)
condition = "(client_id = %s AND path_type = %s AND path_id = %s)"
query = """
UPDATE client_paths
FORCE INDEX (PRIMARY)
SET last_hash_entry_timestamp = FROM_UNIXTIME(%s)
WHERE {}
""".format(" OR ".join([condition] * len(hash_entry_values)))
params = [mysql_utils.RDFDatetimeToTimestamp(now)
] + hash_entry_keys
cursor.execute(query, params)
