def MultiWritePathHistory(self, client_path_histories): """Writes a collection of hash and stat entries observed for given paths.""" for client_path, client_path_history in iteritems( client_path_histories): if client_path.client_id not in self.metadatas: raise db.UnknownClientError(client_path.client_id) path_info = rdf_objects.PathInfo(path_type=client_path.path_type, components=client_path.components) for timestamp, stat_entry in iteritems( client_path_history.stat_entries): path_record = self._GetPathRecord(client_path.client_id, path_info, set_default=False) if path_record is None: # TODO(hanuszczak): Provide more details about paths that caused that. raise db.AtLeastOneUnknownPathError([]) path_record.AddStatEntry(stat_entry, timestamp) for timestamp, hash_entry in iteritems( client_path_history.hash_entries): path_record = self._GetPathRecord(client_path.client_id, path_info, set_default=False) if path_record is None: # TODO(hanuszczak): Provide more details about paths that caused that. raise db.AtLeastOneUnknownPathError([]) path_record.AddHashEntry(hash_entry, timestamp)
def WriteClientCrashInfo(self, client_id, crash_info, cursor=None): """Writes a new client crash record.""" cursor.execute("SET @now = NOW(6)") params = { "client_id": db_utils.ClientIDToInt(client_id), "crash_info": crash_info.SerializeToBytes(), } try: cursor.execute( """ INSERT INTO client_crash_history (client_id, timestamp, crash_info) VALUES (%(client_id)s, @now, %(crash_info)s) """, params) cursor.execute( """ UPDATE clients SET last_crash_timestamp = @now WHERE client_id = %(client_id)s """, params) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def DeleteClient(self, client_id): """Deletes a client with all associated metadata.""" try: del self.metadatas[client_id] except KeyError: raise db.UnknownClientError(client_id) self.clients.pop(client_id, None) self.labels.pop(client_id, None) self.startup_history.pop(client_id, None) self.crash_history.pop(client_id, None) self.client_stats.pop(client_id, None) for key in [k for k in self.flows if k[0] == client_id]: self.flows.pop(key) for key in [k for k in self.flow_requests if k[0] == client_id]: self.flow_requests.pop(key) for key in [ k for k in self.flow_processing_requests if k[0] == client_id ]: self.flow_processing_requests.pop(key) for key in [ k for k in self.client_action_requests if k[0] == client_id ]: self.client_action_requests.pop(key) for kw in self.keywords: self.keywords[kw].pop(client_id, None)
def WriteClientStats(self, client_id: Text, stats: rdf_client_stats.ClientStats, cursor=None) -> None: """Stores a ClientStats instance.""" if stats.timestamp is None: stats.timestamp = rdfvalue.RDFDatetime.Now() try: cursor.execute( """ INSERT INTO client_stats (client_id, payload, timestamp) VALUES (%s, %s, FROM_UNIXTIME(%s)) ON DUPLICATE KEY UPDATE payload=VALUES(payload) """, [ db_utils.ClientIDToInt(client_id), stats.SerializeToBytes(), mysql_utils.RDFDatetimeToTimestamp(stats.timestamp) ]) except MySQLdb.IntegrityError as e: if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2: raise db.UnknownClientError(client_id, cause=e) else: raise
def WriteClientStats(self, client_id, stats): """Stores a ClientStats instance.""" if client_id not in collection.Flatten(self.ReadAllClientIDs()): raise db.UnknownClientError(client_id) self.client_stats[client_id][rdfvalue.RDFDatetime.Now()] = stats
def WriteClientSnapshotHistory(self, clients, cursor=None): """Writes the full history for a particular client.""" client_id = clients[0].client_id latest_timestamp = max(client.timestamp for client in clients) base_params = { "client_id": db_utils.ClientIDToInt(client_id), "latest_timestamp": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp) } try: for client in clients: startup_info = client.startup_info client.startup_info = None params = base_params.copy() params.update({ "timestamp": mysql_utils.RDFDatetimeToTimestamp(client.timestamp), "client_snapshot": client.SerializeToString(), "startup_info": startup_info.SerializeToString(), }) cursor.execute( """ INSERT INTO client_snapshot_history (client_id, timestamp, client_snapshot) VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp)s), %(client_snapshot)s) """, params) cursor.execute( """ INSERT INTO client_startup_history (client_id, timestamp, startup_info) VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp)s), %(startup_info)s) """, params) client.startup_info = startup_info cursor.execute( """ UPDATE clients SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s) WHERE client_id = %(client_id)s AND (last_snapshot_timestamp IS NULL OR last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s)) """, base_params) cursor.execute( """ UPDATE clients SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s) WHERE client_id = %(client_id)s AND (last_startup_timestamp IS NULL OR last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s)) """, base_params) except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id, cause=error)
def WriteFlowObject(self, flow_obj): """Writes a flow object to the database.""" if flow_obj.client_id not in self.metadatas: raise db.UnknownClientError(flow_obj.client_id) clone = flow_obj.Copy() clone.last_update_time = rdfvalue.RDFDatetime.Now() self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone
def AddClientKeywords(self, client_id, keywords): """Associates the provided keywords with the client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) for kw in keywords: self.keywords.setdefault(kw, {}) self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()
def AddClientLabels(self, client_id, owner, labels): """Attaches a user label to a client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set()) for l in labels: labelset.add(utils.SmartUnicode(l))
def WriteClientStartupInfo(self, client_id, startup_info): """Writes a new client startup record.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) ts = rdfvalue.RDFDatetime.Now() self.metadatas[client_id]["startup_info_timestamp"] = ts history = self.startup_history.setdefault(client_id, {}) history[ts] = startup_info.SerializeToBytes()
def WriteClientCrashInfo(self, client_id, crash_info): """Writes a new client crash record.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) ts = rdfvalue.RDFDatetime.Now() self.metadatas[client_id]["last_crash_timestamp"] = ts history = self.crash_history.setdefault(client_id, {}) history[ts] = crash_info.SerializeToBytes()
def WriteClientSnapshotHistory(self, clients, cursor=None): """Writes the full history for a particular client.""" client_id = clients[0].client_id latest_timestamp = max(client.timestamp for client in clients) query = "" params = { "client_id": db_utils.ClientIDToInt(client_id), "latest_timestamp": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp) } for idx, client in enumerate(clients): startup_info = client.startup_info client.startup_info = None query += """ INSERT INTO client_snapshot_history (client_id, timestamp, client_snapshot) VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s), %(client_snapshot_{idx})s); INSERT INTO client_startup_history (client_id, timestamp, startup_info) VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s), %(startup_info_{idx})s); """.format(idx=idx) params.update({ "timestamp_{idx}".format(idx=idx): mysql_utils.RDFDatetimeToTimestamp(client.timestamp), "client_snapshot_{idx}".format(idx=idx): client.SerializeToString(), "startup_info_{idx}".format(idx=idx): startup_info.SerializeToString(), }) client.startup_info = startup_info query += """ UPDATE clients SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s) WHERE client_id = %(client_id)s AND (last_snapshot_timestamp IS NULL OR last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s)); UPDATE clients SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s) WHERE client_id = %(client_id)s AND (last_startup_timestamp IS NULL OR last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s)); """ try: cursor.execute(query, params) except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id, cause=error)
def WriteClientStats(self, client_id, stats): """Stores a ClientStats instance.""" if client_id not in collection.Flatten(self.ReadAllClientIDs()): raise db.UnknownClientError(client_id) if stats.timestamp is None: stats.timestamp = rdfvalue.RDFDatetime.Now() copy = rdf_client_stats.ClientStats(stats) self.client_stats[client_id][copy.timestamp] = copy
def WriteClientSnapshot(self, snapshot, cursor=None): """Write new client snapshot.""" insert_history_query = ( "INSERT INTO client_snapshot_history(client_id, timestamp, " "client_snapshot) VALUES (%s, FROM_UNIXTIME(%s), %s)") insert_startup_query = ( "INSERT INTO client_startup_history(client_id, timestamp, " "startup_info) VALUES(%s, FROM_UNIXTIME(%s), %s)") now = rdfvalue.RDFDatetime.Now() current_timestamp = mysql_utils.RDFDatetimeToTimestamp(now) client_info = { "last_snapshot_timestamp": current_timestamp, "last_startup_timestamp": current_timestamp, "last_version_string": snapshot.GetGRRVersionString(), "last_platform": snapshot.knowledge_base.os, "last_platform_release": snapshot.Uname(), } update_clauses = [ "last_snapshot_timestamp = FROM_UNIXTIME(%(last_snapshot_timestamp)s)", "last_startup_timestamp = FROM_UNIXTIME(%(last_startup_timestamp)s)", "last_version_string = %(last_version_string)s", "last_platform = %(last_platform)s", "last_platform_release = %(last_platform_release)s", ] update_query = ( "UPDATE clients SET {} WHERE client_id = %(client_id)s".format( ", ".join(update_clauses))) int_client_id = db_utils.ClientIDToInt(snapshot.client_id) client_info["client_id"] = int_client_id startup_info = snapshot.startup_info snapshot.startup_info = None try: cursor.execute(insert_history_query, (int_client_id, current_timestamp, snapshot.SerializeToBytes())) cursor.execute(insert_startup_query, (int_client_id, current_timestamp, startup_info.SerializeToBytes())) cursor.execute(update_query, client_info) except MySQLdb.IntegrityError as e: if e.args and e.args[ 0] == mysql_error_constants.NO_REFERENCED_ROW_2: raise db.UnknownClientError(snapshot.client_id, cause=e) elif e.args and e.args[0] == mysql_error_constants.DUP_ENTRY: raise mysql_utils.RetryableError(cause=e) else: raise finally: snapshot.startup_info = startup_info
def _WritePathInfo(self, client_id, path_info): """Writes a single path info record for given client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) path_record = self._GetPathRecord(client_id, path_info) path_record.AddPathInfo(path_info) parent_path_info = path_info.GetParent() if parent_path_info is not None: parent_path_record = self._GetPathRecord(client_id, parent_path_info) parent_path_record.AddChild(path_info)
def WriteScheduledFlow( self, scheduled_flow: rdf_flow_objects.ScheduledFlow) -> None: """See base class.""" if scheduled_flow.client_id not in self.metadatas: raise db.UnknownClientError(scheduled_flow.client_id) if scheduled_flow.creator not in self.users: raise db.UnknownGRRUserError(scheduled_flow.creator) full_id = (scheduled_flow.client_id, scheduled_flow.creator, scheduled_flow.scheduled_flow_id) self.scheduled_flows[full_id] = scheduled_flow.Copy()
def WriteFlowObject(self, flow_obj, allow_update=True): """Writes a flow object to the database.""" if flow_obj.client_id not in self.metadatas: raise db.UnknownClientError(flow_obj.client_id) key = (flow_obj.client_id, flow_obj.flow_id) if not allow_update and key in self.flows: raise db.FlowExistsError(flow_obj.client_id, flow_obj.flow_id) clone = flow_obj.Copy() clone.last_update_time = rdfvalue.RDFDatetime.Now() self.flows[key] = clone
def AddClientLabels(self, client_id, owner, labels, cursor=None): """Attaches a list of user labels to a client.""" cid = db_utils.ClientIDToInt(client_id) labels = set(labels) args = [(cid, mysql_utils.Hash(owner), owner, label) for label in labels] args = list(collection.Flatten(args)) query = """ INSERT IGNORE INTO client_labels (client_id, owner_username_hash, owner_username, label) VALUES {} """.format(", ".join(["(%s, %s, %s, %s)"] * len(labels))) try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshotHistory(self, clients): """Writes the full history for a particular client.""" if clients[0].client_id not in self.metadatas: raise db.UnknownClientError(clients[0].client_id) for client in clients: startup_info = client.startup_info client.startup_info = None snapshots = self.clients.setdefault(client.client_id, {}) snapshots[client.timestamp] = client.SerializeToString() startup_infos = self.startup_history.setdefault(client.client_id, {}) startup_infos[client.timestamp] = startup_info.SerializeToString() client.startup_info = startup_info
def AddClientKeywords(self, client_id, keywords, cursor=None): """Associates the provided keywords with the client.""" cid = db_utils.ClientIDToInt(client_id) keywords = set(keywords) args = [(cid, mysql_utils.Hash(kw), kw) for kw in keywords] args = list(collection.Flatten(args)) query = """ INSERT INTO client_keywords (client_id, keyword_hash, keyword) VALUES {} ON DUPLICATE KEY UPDATE timestamp = NOW(6) """.format(", ".join(["(%s, %s, %s)"] * len(keywords))) try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshot(self, snapshot): """Writes new client snapshot.""" client_id = snapshot.client_id if client_id not in self.metadatas: raise db.UnknownClientError(client_id) startup_info = snapshot.startup_info snapshot.startup_info = None ts = rdfvalue.RDFDatetime.Now() history = self.clients.setdefault(client_id, {}) history[ts] = snapshot.SerializeToBytes() history = self.startup_history.setdefault(client_id, {}) history[ts] = startup_info.SerializeToBytes() snapshot.startup_info = startup_info
def DeleteClient(self, client_id, cursor=None): """Deletes a client with all associated metadata.""" cursor.execute("SELECT COUNT(*) FROM clients WHERE client_id = %s", [db_utils.ClientIDToInt(client_id)]) if cursor.fetchone()[0] == 0: raise db.UnknownClientError(client_id) # Clean out foreign keys first. cursor.execute( """ UPDATE clients SET last_crash_timestamp = NULL, last_snapshot_timestamp = NULL, last_startup_timestamp = NULL WHERE client_id = %s""", [db_utils.ClientIDToInt(client_id)]) cursor.execute("DELETE FROM clients WHERE client_id = %s", [db_utils.ClientIDToInt(client_id)])
def WriteClientStartupInfo(self, client_id, startup_info, cursor=None): """Writes a new client startup record.""" query = """ SET @now = NOW(6); INSERT INTO client_startup_history (client_id, timestamp, startup_info) VALUES (%(client_id)s, @now, %(startup_info)s); UPDATE clients SET last_startup_timestamp = @now WHERE client_id = %(client_id)s; """ params = { "client_id": db_utils.ClientIDToInt(client_id), "startup_info": startup_info.SerializeToString(), } try: cursor.execute(query, params) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def WritePathInfos(self, client_id, path_infos): """Writes a collection of path_info records for a client.""" try: self._MultiWritePathInfos({client_id: path_infos}) except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id=client_id, cause=error)
def WritePathInfos( self, client_id: str, path_infos: Sequence[rdf_objects.PathInfo], cursor: Optional[MySQLdb.cursors.Cursor] = None, ) -> None: """Writes a collection of path_info records for a client.""" now = rdfvalue.RDFDatetime.Now() int_client_id = db_utils.ClientIDToInt(client_id) path_info_values = [] parent_path_info_values = [] stat_entry_keys = [] stat_entry_values = [] hash_entry_keys = [] hash_entry_values = [] for path_info in path_infos: path = mysql_utils.ComponentsToPath(path_info.components) key = ( int_client_id, int(path_info.path_type), path_info.GetPathID().AsBytes(), ) path_info_values.append(key + (mysql_utils.RDFDatetimeToTimestamp(now), path, bool(path_info.directory), len(path_info.components))) if path_info.HasField("stat_entry"): stat_entry_keys.extend(key) stat_entry_values.append( key + (mysql_utils.RDFDatetimeToTimestamp(now), path_info.stat_entry.SerializeToBytes())) if path_info.HasField("hash_entry"): hash_entry_keys.extend(key) hash_entry_values.append( key + (mysql_utils.RDFDatetimeToTimestamp(now), path_info.hash_entry.SerializeToBytes(), path_info.hash_entry.sha256.AsBytes())) # TODO(hanuszczak): Implement a trie in order to avoid inserting # duplicated records. for parent_path_info in path_info.GetAncestors(): path = mysql_utils.ComponentsToPath( parent_path_info.components) parent_path_info_values.append(( int_client_id, int(parent_path_info.path_type), parent_path_info.GetPathID().AsBytes(), path, len(parent_path_info.components), )) if path_info_values: query = """ INSERT INTO client_paths(client_id, path_type, path_id, timestamp, path, directory, depth) VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s, %s) ON DUPLICATE KEY UPDATE timestamp = VALUES(timestamp), directory = directory OR VALUES(directory) """ try: cursor.executemany(query, path_info_values) except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id=client_id, cause=error) if parent_path_info_values: query = """ INSERT INTO client_paths(client_id, path_type, path_id, path, directory, depth) VALUES (%s, %s, %s, %s, TRUE, %s) ON DUPLICATE KEY UPDATE directory = TRUE, timestamp = NOW(6) """ cursor.executemany(query, parent_path_info_values) if stat_entry_values: query = """ INSERT INTO client_path_stat_entries(client_id, path_type, path_id, timestamp, stat_entry) VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s) """ cursor.executemany(query, stat_entry_values) condition = "(client_id = %s AND path_type = %s AND path_id = %s)" query = """ UPDATE client_paths FORCE INDEX (PRIMARY) SET last_stat_entry_timestamp = FROM_UNIXTIME(%s) WHERE {} """.format(" OR ".join([condition] * len(stat_entry_values))) params = [mysql_utils.RDFDatetimeToTimestamp(now) ] + stat_entry_keys cursor.execute(query, params) if hash_entry_values: query = """ INSERT INTO client_path_hash_entries(client_id, path_type, path_id, timestamp, hash_entry, sha256) VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s) """ cursor.executemany(query, hash_entry_values) condition = "(client_id = %s AND path_type = %s AND path_id = %s)" query = """ UPDATE client_paths FORCE INDEX (PRIMARY) SET last_hash_entry_timestamp = FROM_UNIXTIME(%s) WHERE {} """.format(" OR ".join([condition] * len(hash_entry_values))) params = [mysql_utils.RDFDatetimeToTimestamp(now) ] + hash_entry_keys cursor.execute(query, params)