def WriteFlowObject(self, flow_obj, cursor=None):
"""Writes a flow object to the database."""
query = ("INSERT INTO flows "
"(client_id, flow_id, long_flow_id, parent_flow_id, flow, "
"next_request_to_process, timestamp, last_update) VALUES "
"(%s, %s, %s, %s, %s, %s, %s, %s) "
"ON DUPLICATE KEY UPDATE "
"flow=VALUES(flow), "
"next_request_to_process=VALUES(next_request_to_process),"
"last_update=VALUES(last_update)")
if flow_obj.parent_flow_id:
pfi = mysql_utils.FlowIDToInt(flow_obj.parent_flow_id)
else:
pfi = None
timestamp_str = mysql_utils.RDFDatetimeToMysqlString(flow_obj.create_time)
now_str = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now())
args = [
mysql_utils.ClientIDToInt(flow_obj.client_id),
mysql_utils.FlowIDToInt(flow_obj.flow_id), flow_obj.long_flow_id, pfi,
flow_obj.SerializeToString(), flow_obj.next_request_to_process,
timestamp_str, now_str
]
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(flow_obj.client_id, cause=e)
def WriteClientStats(self, client_id,
stats):
"""Stores a ClientStats instance."""
if client_id not in self.ReadAllClientIDs():
raise db.UnknownClientError(client_id)
self.client_stats[client_id][stats.create_time] = stats
def MultiWritePathHistory(self, client_path_histories):
"""Writes a collection of hash and stat entries observed for given paths."""
for client_path, client_path_history in iteritems(
client_path_histories):
if client_path.client_id not in self.metadatas:
raise db.UnknownClientError(client_path.client_id)
path_info = rdf_objects.PathInfo(path_type=client_path.path_type,
components=client_path.components)
for timestamp, stat_entry in iteritems(
client_path_history.stat_entries):
path_record = self._GetPathRecord(client_path.client_id,
path_info,
set_default=False)
if path_record is None:
# TODO(hanuszczak): Provide more details about paths that caused that.
raise db.AtLeastOneUnknownPathError([])
path_record.AddStatEntry(stat_entry, timestamp)
for timestamp, hash_entry in iteritems(
client_path_history.hash_entries):
path_record = self._GetPathRecord(client_path.client_id,
path_info,
set_default=False)
if path_record is None:
# TODO(hanuszczak): Provide more details about paths that caused that.
raise db.AtLeastOneUnknownPathError([])
path_record.AddHashEntry(hash_entry, timestamp)
def WriteClientSnapshot(self, client, cursor=None):
"""Write new client snapshot."""
startup_info = client.startup_info
client.startup_info = None
insert_history_query = (
"INSERT INTO client_snapshot_history(client_id, timestamp, "
"client_snapshot) VALUES (%s, %s, %s)")
insert_startup_query = (
"INSERT INTO client_startup_history(client_id, timestamp, "
"startup_info) VALUES(%s, %s, %s)")
update_query = ("UPDATE clients SET last_client_timestamp=%s, "
"last_startup_timestamp=%s "
"WHERE client_id = %s")
int_id = mysql_utils.ClientIDToInt(client.client_id)
timestamp = datetime.datetime.utcnow()
try:
cursor.execute(insert_history_query,
(int_id, timestamp, client.SerializeToString()))
cursor.execute(
insert_startup_query,
(int_id, timestamp, startup_info.SerializeToString()))
cursor.execute(update_query, (timestamp, timestamp, int_id))
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client.client_id, cause=e)
finally:
client.startup_info = startup_info
def ReadPathInfosHistories(self, client_id, path_type, components_list):
"""Reads a collection of hash and stat entries for given paths."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
results = {}
for components in components_list:
path_record = self.path_records[(client_id, path_type, components)]
entries_by_ts = {}
for ts, stat_entry in path_record.GetStatEntries():
pi = rdf_objects.PathInfo(path_type=path_type,
components=components,
timestamp=ts,
stat_entry=stat_entry)
entries_by_ts[ts] = pi
for ts, hash_entry in path_record.GetHashEntries():
try:
pi = entries_by_ts[ts]
except KeyError:
pi = rdf_objects.PathInfo(path_type=path_type,
components=components,
timestamp=ts)
entries_by_ts[ts] = pi
pi.hash_entry = hash_entry
results[components] = [
entries_by_ts[k] for k in sorted(entries_by_ts.iterkeys())
]
return results
def AddClientKeywords(self, client_id, keywords):
"""Associates the provided keywords with the client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
for kw in keywords:
self.keywords.setdefault(kw, {})
self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()
def WriteFlowObject(self, flow_obj):
"""Writes a flow object to the database."""
if flow_obj.client_id not in self.metadatas:
raise db.UnknownClientError(flow_obj.client_id)
clone = flow_obj.Copy()
clone.last_update_time = rdfvalue.RDFDatetime.Now()
self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone
def AddClientLabels(self, client_id, owner, labels):
"""Attaches a user label to a client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())
for l in labels:
labelset.add(utils.SmartUnicode(l))
def WriteClientStartupInfo(self, client_id, startup_info):
"""Writes a new client startup record."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
ts = rdfvalue.RDFDatetime.Now()
self.metadatas[client_id]["startup_info_timestamp"] = ts
history = self.startup_history.setdefault(client_id, {})
history[ts] = startup_info.SerializeToString()
def WriteClientCrashInfo(self, client_id, crash_info):
"""Writes a new client crash record."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
ts = rdfvalue.RDFDatetime.Now()
self.metadatas[client_id]["last_crash_timestamp"] = ts
history = self.crash_history.setdefault(client_id, {})
history[ts] = crash_info.SerializeToString()
def AddClientLabels(self, client_id, owner, labels, cursor=None):
"""Attaches a list of user labels to a client."""
cid = mysql_utils.ClientIDToInt(client_id)
try:
for label in labels:
cursor.execute(
"INSERT IGNORE INTO client_labels (client_id, owner, label) "
"VALUES (%s, %s, %s)",
[cid, owner, utils.SmartUnicode(label)])
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def MultiWritePathHistory(self, client_id, stat_entries, hash_entries):
"""Writes a collection of hash and stat entries observed for given paths."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
for path_info, stat_entry in iteritems(stat_entries):
path_record = self._GetPathRecord(client_id, path_info)
path_record.AddStatEntry(stat_entry, path_info.timestamp)
for path_info, hash_entry in iteritems(hash_entries):
path_record = self._GetPathRecord(client_id, path_info)
path_record.AddHashEntry(hash_entry, path_info.timestamp)
def AddClientKeywords(self, client_id, keywords, cursor=None):
"""Associates the provided keywords with the client."""
cid = mysql_utils.ClientIDToInt(client_id)
now = datetime.datetime.utcnow()
try:
for kw in keywords:
cursor.execute(
"INSERT INTO client_keywords (client_id, keyword, timestamp) "
"VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE timestamp=%s",
[cid, utils.SmartUnicode(kw), now, now])
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def _WritePathInfo(self, client_id, path_info):
"""Writes a single path info record for given client."""
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
path_record = self._GetPathRecord(client_id, path_info)
path_record.AddPathInfo(path_info)
parent_path_info = path_info.GetParent()
if parent_path_info is not None:
parent_path_record = self._GetPathRecord(client_id,
parent_path_info)
parent_path_record.AddChild(path_info)
def WriteClientCrashInfo(self, client_id, crash_info, cursor=None):
"""Writes a new client crash record."""
cid = mysql_utils.ClientIDToInt(client_id)
now = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now())
try:
cursor.execute(
"INSERT INTO client_crash_history (client_id, timestamp, crash_info) "
"VALUES (%s, %s, %s)",
[cid, now, crash_info.SerializeToString()])
cursor.execute(
"UPDATE clients SET last_crash_timestamp = %s WHERE client_id=%s",
[now, cid])
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshotHistory(self, clients):
"""Writes the full history for a particular client."""
if clients[0].client_id not in self.metadatas:
raise db.UnknownClientError(clients[0].client_id)
for client in clients:
startup_info = client.startup_info
client.startup_info = None
snapshots = self.clients.setdefault(client.client_id, {})
snapshots[client.timestamp] = client.SerializeToString()
startup_infos = self.startup_history.setdefault(client.client_id, {})
startup_infos[client.timestamp] = startup_info.SerializeToString()
client.startup_info = startup_info
def AddClientLabels(self, client_id, owner, labels, cursor=None):
"""Attaches a list of user labels to a client."""
cid = mysql_utils.ClientIDToInt(client_id)
labels = set(labels)
args = [(cid, mysql_utils.Hash(owner), owner, label)
for label in labels]
args = list(collection.Flatten(args))
query = """
INSERT IGNORE INTO client_labels
(client_id, owner_username_hash, owner_username, label)
VALUES {}
""".format(", ".join(["(%s, %s, %s, %s)"] * len(labels)))
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def AddClientKeywords(self, client_id, keywords, cursor=None):
"""Associates the provided keywords with the client."""
cid = mysql_utils.ClientIDToInt(client_id)
now = datetime.datetime.utcnow()
keywords = set(keywords)
args = [(cid, mysql_utils.Hash(kw), kw, now) for kw in keywords]
args = list(collection.Flatten(args))
query = """
INSERT INTO client_keywords
(client_id, keyword_hash, keyword, timestamp)
VALUES {}
ON DUPLICATE KEY UPDATE timestamp = VALUES(timestamp)
""".format(", ".join(["(%s, %s, %s, %s)"] * len(keywords)))
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshot(self, client):
"""Writes new client snapshot."""
client_id = client.client_id
if client_id not in self.metadatas:
raise db.UnknownClientError(client_id)
startup_info = client.startup_info
client.startup_info = None
ts = rdfvalue.RDFDatetime.Now()
history = self.clients.setdefault(client_id, {})
history[ts] = client.SerializeToString()
history = self.startup_history.setdefault(client_id, {})
history[ts] = startup_info.SerializeToString()
client.startup_info = startup_info
def WriteClientSnapshot(self, snapshot, cursor=None):
"""Write new client snapshot."""
insert_history_query = (
"INSERT INTO client_snapshot_history(client_id, timestamp, "
"client_snapshot) VALUES (%s, %s, %s)")
insert_startup_query = (
"INSERT INTO client_startup_history(client_id, timestamp, "
"startup_info) VALUES(%s, %s, %s)")
client_platform = snapshot.knowledge_base.os
current_timestamp = datetime.datetime.utcnow()
client_info = {
"last_client_timestamp": current_timestamp,
"last_startup_timestamp": current_timestamp,
"last_version_string": snapshot.GetGRRVersionString(),
"last_platform_release": snapshot.Uname(),
}
if client_platform:
client_info["last_platform"] = client_platform
update_clauses = [
"{0} = %({0})s".format(k) for k in iterkeys(client_info)
]
update_query = (
"UPDATE clients SET {} WHERE client_id = %(client_id)s".format(
", ".join(update_clauses)))
int_client_id = mysql_utils.ClientIDToInt(snapshot.client_id)
client_info["client_id"] = int_client_id
startup_info = snapshot.startup_info
snapshot.startup_info = None
try:
cursor.execute(insert_history_query,
(int_client_id, current_timestamp,
snapshot.SerializeToString()))
cursor.execute(insert_startup_query,
(int_client_id, current_timestamp,
startup_info.SerializeToString()))
cursor.execute(update_query, client_info)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(snapshot.client_id, cause=e)
finally:
snapshot.startup_info = startup_info
def WriteClientStats(self, client_id, stats, cursor=None):
"""Stores a ClientStats instance."""
try:
cursor.execute(
"""
INSERT INTO client_stats (client_id, payload, timestamp)
VALUES (%s, %s, %s)
ON DUPLICATE KEY UPDATE payload=VALUES(payload)
""", [
mysql_utils.ClientIDToInt(client_id),
stats.SerializeToString(),
mysql_utils.RDFDatetimeToMysqlString(stats.create_time)
])
except MySQLdb.IntegrityError as e:
if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2:
raise db.UnknownClientError(client_id, cause=e)
else:
raise
def WriteClientSnapshotHistory(self, clients, cursor=None):
"""Writes the full history for a particular client."""
cid = mysql_utils.ClientIDToInt(clients[0].client_id)
latest_timestamp = None
for client in clients:
startup_info = client.startup_info
client.startup_info = None
timestamp = mysql_utils.RDFDatetimeToMysqlString(client.timestamp)
latest_timestamp = max(latest_timestamp, client.timestamp)
try:
cursor.execute(
"INSERT INTO client_snapshot_history "
"(client_id, timestamp, client_snapshot) "
"VALUES (%s, %s, %s)",
[cid, timestamp,
client.SerializeToString()])
cursor.execute(
"INSERT INTO client_startup_history "
"(client_id, timestamp, startup_info) "
"VALUES (%s, %s, %s)",
[cid, timestamp,
startup_info.SerializeToString()])
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(clients[0].client_id, cause=e)
finally:
client.startup_info = startup_info
latest_timestamp_str = mysql_utils.RDFDatetimeToMysqlString(
latest_timestamp)
cursor.execute(
"UPDATE clients SET last_client_timestamp=%s "
"WHERE client_id = %s AND "
"(last_client_timestamp IS NULL OR last_client_timestamp < %s)",
[latest_timestamp_str, cid, latest_timestamp_str])
cursor.execute(
"UPDATE clients SET last_startup_timestamp=%s "
"WHERE client_id = %s AND "
"(last_startup_timestamp IS NULL OR last_startup_timestamp < %s)",
[latest_timestamp_str, cid, latest_timestamp_str])
def WriteClientMessages(self, messages, cursor=None):
"""Writes messages that should go to the client to the db."""
query = ("INSERT IGNORE INTO client_messages "
"(client_id, message_id, timestamp, message) "
"VALUES %s ON DUPLICATE KEY UPDATE "
"timestamp=VALUES(timestamp), message=VALUES(message)")
now = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now())
value_templates = []
args = []
for m in messages:
client_id_int = mysql_utils.ClientIDToInt(
db_utils.ClientIdFromGrrMessage(m))
args.extend([client_id_int, m.task_id, now, m.SerializeToString()])
value_templates.append("(%s, %s, %s, %s)")
query %= ",".join(value_templates)
try:
cursor.execute(query, args)
except MySQLdb.IntegrityError as e:
raise db.UnknownClientError(cause=e)
def WritePathInfos(self, client_id, path_infos):
"""Writes a collection of path_info records for a client."""
try:
self._MultiWritePathInfos({client_id: path_infos})
except MySQLdb.IntegrityError as error:
raise db.UnknownClientError(client_id=client_id, cause=error)
def WriteFlowObject(self, flow_obj):
"""Writes a flow object to the database."""
if flow_obj.client_id not in self.metadatas:
raise db.UnknownClientError(flow_obj.client_id)
self.flows[(flow_obj.client_id, flow_obj.flow_id)] = flow_obj.Copy()
