def WriteFlowObject(self, flow_obj, cursor=None): """Writes a flow object to the database.""" query = ("INSERT INTO flows " "(client_id, flow_id, long_flow_id, parent_flow_id, flow, " "next_request_to_process, timestamp, last_update) VALUES " "(%s, %s, %s, %s, %s, %s, %s, %s) " "ON DUPLICATE KEY UPDATE " "flow=VALUES(flow), " "next_request_to_process=VALUES(next_request_to_process)," "last_update=VALUES(last_update)") if flow_obj.parent_flow_id: pfi = mysql_utils.FlowIDToInt(flow_obj.parent_flow_id) else: pfi = None timestamp_str = mysql_utils.RDFDatetimeToMysqlString(flow_obj.create_time) now_str = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now()) args = [ mysql_utils.ClientIDToInt(flow_obj.client_id), mysql_utils.FlowIDToInt(flow_obj.flow_id), flow_obj.long_flow_id, pfi, flow_obj.SerializeToString(), flow_obj.next_request_to_process, timestamp_str, now_str ] try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(flow_obj.client_id, cause=e)
def WriteClientStats(self, client_id, stats): """Stores a ClientStats instance.""" if client_id not in self.ReadAllClientIDs(): raise db.UnknownClientError(client_id) self.client_stats[client_id][stats.create_time] = stats
def MultiWritePathHistory(self, client_path_histories): """Writes a collection of hash and stat entries observed for given paths.""" for client_path, client_path_history in iteritems( client_path_histories): if client_path.client_id not in self.metadatas: raise db.UnknownClientError(client_path.client_id) path_info = rdf_objects.PathInfo(path_type=client_path.path_type, components=client_path.components) for timestamp, stat_entry in iteritems( client_path_history.stat_entries): path_record = self._GetPathRecord(client_path.client_id, path_info, set_default=False) if path_record is None: # TODO(hanuszczak): Provide more details about paths that caused that. raise db.AtLeastOneUnknownPathError([]) path_record.AddStatEntry(stat_entry, timestamp) for timestamp, hash_entry in iteritems( client_path_history.hash_entries): path_record = self._GetPathRecord(client_path.client_id, path_info, set_default=False) if path_record is None: # TODO(hanuszczak): Provide more details about paths that caused that. raise db.AtLeastOneUnknownPathError([]) path_record.AddHashEntry(hash_entry, timestamp)
def WriteClientSnapshot(self, client, cursor=None): """Write new client snapshot.""" startup_info = client.startup_info client.startup_info = None insert_history_query = ( "INSERT INTO client_snapshot_history(client_id, timestamp, " "client_snapshot) VALUES (%s, %s, %s)") insert_startup_query = ( "INSERT INTO client_startup_history(client_id, timestamp, " "startup_info) VALUES(%s, %s, %s)") update_query = ("UPDATE clients SET last_client_timestamp=%s, " "last_startup_timestamp=%s " "WHERE client_id = %s") int_id = mysql_utils.ClientIDToInt(client.client_id) timestamp = datetime.datetime.utcnow() try: cursor.execute(insert_history_query, (int_id, timestamp, client.SerializeToString())) cursor.execute( insert_startup_query, (int_id, timestamp, startup_info.SerializeToString())) cursor.execute(update_query, (timestamp, timestamp, int_id)) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client.client_id, cause=e) finally: client.startup_info = startup_info
def ReadPathInfosHistories(self, client_id, path_type, components_list): """Reads a collection of hash and stat entries for given paths.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) results = {} for components in components_list: path_record = self.path_records[(client_id, path_type, components)] entries_by_ts = {} for ts, stat_entry in path_record.GetStatEntries(): pi = rdf_objects.PathInfo(path_type=path_type, components=components, timestamp=ts, stat_entry=stat_entry) entries_by_ts[ts] = pi for ts, hash_entry in path_record.GetHashEntries(): try: pi = entries_by_ts[ts] except KeyError: pi = rdf_objects.PathInfo(path_type=path_type, components=components, timestamp=ts) entries_by_ts[ts] = pi pi.hash_entry = hash_entry results[components] = [ entries_by_ts[k] for k in sorted(entries_by_ts.iterkeys()) ] return results
def AddClientKeywords(self, client_id, keywords): """Associates the provided keywords with the client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) for kw in keywords: self.keywords.setdefault(kw, {}) self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()
def WriteFlowObject(self, flow_obj): """Writes a flow object to the database.""" if flow_obj.client_id not in self.metadatas: raise db.UnknownClientError(flow_obj.client_id) clone = flow_obj.Copy() clone.last_update_time = rdfvalue.RDFDatetime.Now() self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone
def AddClientLabels(self, client_id, owner, labels): """Attaches a user label to a client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set()) for l in labels: labelset.add(utils.SmartUnicode(l))
def WriteClientStartupInfo(self, client_id, startup_info): """Writes a new client startup record.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) ts = rdfvalue.RDFDatetime.Now() self.metadatas[client_id]["startup_info_timestamp"] = ts history = self.startup_history.setdefault(client_id, {}) history[ts] = startup_info.SerializeToString()
def WriteClientCrashInfo(self, client_id, crash_info): """Writes a new client crash record.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) ts = rdfvalue.RDFDatetime.Now() self.metadatas[client_id]["last_crash_timestamp"] = ts history = self.crash_history.setdefault(client_id, {}) history[ts] = crash_info.SerializeToString()
def AddClientLabels(self, client_id, owner, labels, cursor=None): """Attaches a list of user labels to a client.""" cid = mysql_utils.ClientIDToInt(client_id) try: for label in labels: cursor.execute( "INSERT IGNORE INTO client_labels (client_id, owner, label) " "VALUES (%s, %s, %s)", [cid, owner, utils.SmartUnicode(label)]) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def MultiWritePathHistory(self, client_id, stat_entries, hash_entries): """Writes a collection of hash and stat entries observed for given paths.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) for path_info, stat_entry in iteritems(stat_entries): path_record = self._GetPathRecord(client_id, path_info) path_record.AddStatEntry(stat_entry, path_info.timestamp) for path_info, hash_entry in iteritems(hash_entries): path_record = self._GetPathRecord(client_id, path_info) path_record.AddHashEntry(hash_entry, path_info.timestamp)
def AddClientKeywords(self, client_id, keywords, cursor=None): """Associates the provided keywords with the client.""" cid = mysql_utils.ClientIDToInt(client_id) now = datetime.datetime.utcnow() try: for kw in keywords: cursor.execute( "INSERT INTO client_keywords (client_id, keyword, timestamp) " "VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE timestamp=%s", [cid, utils.SmartUnicode(kw), now, now]) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def _WritePathInfo(self, client_id, path_info): """Writes a single path info record for given client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) path_record = self._GetPathRecord(client_id, path_info) path_record.AddPathInfo(path_info) parent_path_info = path_info.GetParent() if parent_path_info is not None: parent_path_record = self._GetPathRecord(client_id, parent_path_info) parent_path_record.AddChild(path_info)
def WriteClientCrashInfo(self, client_id, crash_info, cursor=None): """Writes a new client crash record.""" cid = mysql_utils.ClientIDToInt(client_id) now = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now()) try: cursor.execute( "INSERT INTO client_crash_history (client_id, timestamp, crash_info) " "VALUES (%s, %s, %s)", [cid, now, crash_info.SerializeToString()]) cursor.execute( "UPDATE clients SET last_crash_timestamp = %s WHERE client_id=%s", [now, cid]) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshotHistory(self, clients): """Writes the full history for a particular client.""" if clients[0].client_id not in self.metadatas: raise db.UnknownClientError(clients[0].client_id) for client in clients: startup_info = client.startup_info client.startup_info = None snapshots = self.clients.setdefault(client.client_id, {}) snapshots[client.timestamp] = client.SerializeToString() startup_infos = self.startup_history.setdefault(client.client_id, {}) startup_infos[client.timestamp] = startup_info.SerializeToString() client.startup_info = startup_info
def AddClientLabels(self, client_id, owner, labels, cursor=None): """Attaches a list of user labels to a client.""" cid = mysql_utils.ClientIDToInt(client_id) labels = set(labels) args = [(cid, mysql_utils.Hash(owner), owner, label) for label in labels] args = list(collection.Flatten(args)) query = """ INSERT IGNORE INTO client_labels (client_id, owner_username_hash, owner_username, label) VALUES {} """.format(", ".join(["(%s, %s, %s, %s)"] * len(labels))) try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def AddClientKeywords(self, client_id, keywords, cursor=None): """Associates the provided keywords with the client.""" cid = mysql_utils.ClientIDToInt(client_id) now = datetime.datetime.utcnow() keywords = set(keywords) args = [(cid, mysql_utils.Hash(kw), kw, now) for kw in keywords] args = list(collection.Flatten(args)) query = """ INSERT INTO client_keywords (client_id, keyword_hash, keyword, timestamp) VALUES {} ON DUPLICATE KEY UPDATE timestamp = VALUES(timestamp) """.format(", ".join(["(%s, %s, %s, %s)"] * len(keywords))) try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(client_id, cause=e)
def WriteClientSnapshot(self, client): """Writes new client snapshot.""" client_id = client.client_id if client_id not in self.metadatas: raise db.UnknownClientError(client_id) startup_info = client.startup_info client.startup_info = None ts = rdfvalue.RDFDatetime.Now() history = self.clients.setdefault(client_id, {}) history[ts] = client.SerializeToString() history = self.startup_history.setdefault(client_id, {}) history[ts] = startup_info.SerializeToString() client.startup_info = startup_info
def WriteClientSnapshot(self, snapshot, cursor=None): """Write new client snapshot.""" insert_history_query = ( "INSERT INTO client_snapshot_history(client_id, timestamp, " "client_snapshot) VALUES (%s, %s, %s)") insert_startup_query = ( "INSERT INTO client_startup_history(client_id, timestamp, " "startup_info) VALUES(%s, %s, %s)") client_platform = snapshot.knowledge_base.os current_timestamp = datetime.datetime.utcnow() client_info = { "last_client_timestamp": current_timestamp, "last_startup_timestamp": current_timestamp, "last_version_string": snapshot.GetGRRVersionString(), "last_platform_release": snapshot.Uname(), } if client_platform: client_info["last_platform"] = client_platform update_clauses = [ "{0} = %({0})s".format(k) for k in iterkeys(client_info) ] update_query = ( "UPDATE clients SET {} WHERE client_id = %(client_id)s".format( ", ".join(update_clauses))) int_client_id = mysql_utils.ClientIDToInt(snapshot.client_id) client_info["client_id"] = int_client_id startup_info = snapshot.startup_info snapshot.startup_info = None try: cursor.execute(insert_history_query, (int_client_id, current_timestamp, snapshot.SerializeToString())) cursor.execute(insert_startup_query, (int_client_id, current_timestamp, startup_info.SerializeToString())) cursor.execute(update_query, client_info) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(snapshot.client_id, cause=e) finally: snapshot.startup_info = startup_info
def WriteClientStats(self, client_id, stats, cursor=None): """Stores a ClientStats instance.""" try: cursor.execute( """ INSERT INTO client_stats (client_id, payload, timestamp) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE payload=VALUES(payload) """, [ mysql_utils.ClientIDToInt(client_id), stats.SerializeToString(), mysql_utils.RDFDatetimeToMysqlString(stats.create_time) ]) except MySQLdb.IntegrityError as e: if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2: raise db.UnknownClientError(client_id, cause=e) else: raise
def WriteClientSnapshotHistory(self, clients, cursor=None): """Writes the full history for a particular client.""" cid = mysql_utils.ClientIDToInt(clients[0].client_id) latest_timestamp = None for client in clients: startup_info = client.startup_info client.startup_info = None timestamp = mysql_utils.RDFDatetimeToMysqlString(client.timestamp) latest_timestamp = max(latest_timestamp, client.timestamp) try: cursor.execute( "INSERT INTO client_snapshot_history " "(client_id, timestamp, client_snapshot) " "VALUES (%s, %s, %s)", [cid, timestamp, client.SerializeToString()]) cursor.execute( "INSERT INTO client_startup_history " "(client_id, timestamp, startup_info) " "VALUES (%s, %s, %s)", [cid, timestamp, startup_info.SerializeToString()]) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(clients[0].client_id, cause=e) finally: client.startup_info = startup_info latest_timestamp_str = mysql_utils.RDFDatetimeToMysqlString( latest_timestamp) cursor.execute( "UPDATE clients SET last_client_timestamp=%s " "WHERE client_id = %s AND " "(last_client_timestamp IS NULL OR last_client_timestamp < %s)", [latest_timestamp_str, cid, latest_timestamp_str]) cursor.execute( "UPDATE clients SET last_startup_timestamp=%s " "WHERE client_id = %s AND " "(last_startup_timestamp IS NULL OR last_startup_timestamp < %s)", [latest_timestamp_str, cid, latest_timestamp_str])
def WriteClientMessages(self, messages, cursor=None): """Writes messages that should go to the client to the db.""" query = ("INSERT IGNORE INTO client_messages " "(client_id, message_id, timestamp, message) " "VALUES %s ON DUPLICATE KEY UPDATE " "timestamp=VALUES(timestamp), message=VALUES(message)") now = mysql_utils.RDFDatetimeToMysqlString(rdfvalue.RDFDatetime.Now()) value_templates = [] args = [] for m in messages: client_id_int = mysql_utils.ClientIDToInt( db_utils.ClientIdFromGrrMessage(m)) args.extend([client_id_int, m.task_id, now, m.SerializeToString()]) value_templates.append("(%s, %s, %s, %s)") query %= ",".join(value_templates) try: cursor.execute(query, args) except MySQLdb.IntegrityError as e: raise db.UnknownClientError(cause=e)
def WritePathInfos(self, client_id, path_infos): """Writes a collection of path_info records for a client.""" try: self._MultiWritePathInfos({client_id: path_infos}) except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id=client_id, cause=error)
def WriteFlowObject(self, flow_obj): """Writes a flow object to the database.""" if flow_obj.client_id not in self.metadatas: raise db.UnknownClientError(flow_obj.client_id) self.flows[(flow_obj.client_id, flow_obj.flow_id)] = flow_obj.Copy()