def get_file_data(app_id, agent_id=None): conn = db_connect() try: if agent_id: file_data = list( r.table(FilesCollection).filter(lambda x: ( x[FilesKey.AppIds].contains(app_id) & x[FilesKey.AgentIds].contains(agent_id))).without( FilesKey.AppIds, FilesKey.AgentIds, ).run(conn)) else: file_data = list( r.table(FilesCollection).filter( lambda x: (x[FilesKey.AppIds].contains(app_id))).without( FilesKey.AppIds, FilesKey.AgentIds, ).run(conn)) except Exception as e: file_data = [] logger.exception(e) conn.close() return (file_data)
def delete_mouse(mouse_name, username, uri, method): conn = db_connect() try: ( r .table(RelayServersCollection) .get(mouse_name) .delete() .run(conn) ) status = ( MightyMouseResults( username, uri, method ).remove(mouse_name) ) except Exception as e: status = ( MightyMouseResults( username, uri, method ).failed_to_remove(mouse_name, e) ) logger.exception(status) conn.close() return(status)
def update_customers_in_app(customer_name, app_id, table=AppsCollection): conn = db_connect() if table == AppsCollection: CurrentAppsKey = AppsKey elif table == CustomAppsCollection: CurrentAppsKey = CustomAppsKey elif table == SupportedAppsCollection: CurrentAppsKey = SupportedAppsKey elif table == AgentAppsCollection: CurrentAppsKey = AgentAppsKey try: exists = (r.table(table).get(app_id).run(conn)) if exists: (r.table(table).get(app_id).update({ CurrentAppsKey.Customers: (r.row[CurrentAppsKey.Customers].set_insert(customer_name)), }).run(conn)) except Exception as e: logger.exception(e) conn.close()
def get_windows_bulletinid_and_cveids(kb): conn = db_connect() info = { WindowsSecurityBulletinKey.BulletinId: '', WindowsSecurityBulletinKey.CveIds: [] } try: info = list( r .table(WindowsSecurityBulletinCollection) .get_all(kb, index=WindowsSecurityBulletinIndexes.ComponentKb) .pluck(WindowsSecurityBulletinKey.BulletinId, WindowsSecurityBulletinKey.CveIds) .run(conn) ) if info: if len(info) > 0: info = info[0] logger.debug('retrieved microsoft bulletin info: %s', info) except Exception as e: logger.exception(e) conn.close() return(info)
def update_file_data(app_id, agent_id, file_data): conn = db_connect() try: if len(file_data) > 0: for uri in file_data: exists = (r.table(FilesCollection).get( uri[FilesKey.FileName]).run(conn)) if exists: (r.table(FilesCollection).get( uri[FilesKey.FileName]).update({ FilesKey.AppIds: (r.row[FilesKey.AppIds].set_insert(app_id)), FilesKey.AgentIds: (r.row[FilesKey.AgentIds].set_insert(agent_id)) }).run(conn)) else: (r.table(FilesCollection).insert( { FilesKey.AppIds: [app_id], FilesKey.AgentIds: [agent_id], FilesKey.FileName: uri[FilesKey.FileName], FilesKey.FileSize: uri[FilesKey.FileSize], FilesKey.FileUri: uri[FilesKey.FileUri], FilesKey.FileHash: uri[FilesKey.FileHash], }, ).run(conn, no_reply=True)) except Exception as e: logger.exception(e) conn.close()
def dataReceived(self, data): # foo = encode(data) #bar = decode(data) # loads(foo) # loads(data) print data valid_json = verifyJsonIsValid(data) print valid_json if valid_json[0]: json_data = valid_json[1] self.session = db_connect() self.client_ip = self.transport.getPeer() self.node_exists = nodeExists(self.session, self.client_ip.host) if not self.node_exists: addNode(self.session, self.client_ip.host) self.node_exists = nodeExists(self.session, self.client_ip.host) if json_data[OPERATION] == SYSTEM_INFO: addSystemInfo(self.session, json_data, self.node_exists) self.session.commit() print "session committed" self.session.close() print "session closed" self.transport.write(dumps(json_data))
def update_mouse(exist, mouse_name, customer_names, address, username, uri, method): conn = db_connect() if not address: address = exist[RelayServers.Address] data = { RelayServers.Customers: customer_names, RelayServers.Address: address, } try: a = ( r .table(RelayServersCollection) .get(mouse_name) .update(data) .run(conn) ) status = ( MightyMouseResults( username, uri, method ).updated(mouse_name, data) ) except Exception as e: status = ( MightyMouseResults( username, uri, method ).failed_to_update(mouse_name, e) ) logger.exception(status) conn.close() return(status)
def add_mouse(customer_names, mouse_name, address, username, uri, method): conn = db_connect() data = { RelayServers.RelayName: mouse_name, RelayServers.Customers: customer_names, RelayServers.Address: address, } try: ( r .table(RelayServersCollection) .insert(data) .run(conn) ) status = ( MightyMouseResults( username, uri, method ).created(mouse_name, data) ) except Exception as e: status = ( MightyMouseResults( username, uri, method ).failed_to_create(mouse_name, e) ) logger.exception(status) conn.close() return(status)
def update_agent_status(agentid, username, uri=None, method=None): try: conn = db_connect() update_status = { AgentKey.LastAgentUpdate: r.epoch_time(mktime(datetime.now().timetuple())), AgentKey.AgentStatus: 'up' } exists = (r.table(AgentsCollection).get(agentid).run(conn)) if exists: (r.table(AgentsCollection).get(agentid).update(update_status).run( conn)) status = (GenericResults(username, uri, method).object_updated( agentid, 'agent', update_status)) else: status = (GenericResults(username, uri, method).does_not_exists(agentid, 'agent')) logger.info(status['message']) conn.close() except Exception as e: status = (GenericResults(username, uri, method).something_broke(agentid, 'agent', e)) logger.exception(e)
def get_ubuntu_cveids(app_name, app_version): conn = db_connect() info = { UbuntuSecurityBulletinKey.BulletinId: '', UbuntuSecurityBulletinKey.CveIds: [] } try: info = list( r .table(UbuntuSecurityBulletinCollection) .get_all( [app_name, app_version], index=UbuntuSecurityBulletinIndexes.NameAndVersion ) .pluck(UbuntuSecurityBulletinKey.BulletinId, UbuntuSecurityBulletinKey.CveIds) .run(conn) ) if info: if len(info) > 1: info = info[0] logger.debug('retrieved ubuntu bulletin info: %s', info) except Exception as e: logger.exception(e) return(info)
def sync_supported_updates_to_all_agents(self, apps): try: conn = db_connect() deleted_count = 0 (r.table(self.CurrentAppsPerAgentCollection).delete().run(conn)) conn.close() self.update_agents_with_supported(apps) except Exception as e: logger.exception(e)
def _update_completed_time_on_agents(self, oper_id, agent_id): try: conn = db_connect() (r.table(OperationsPerAgentCollection).get_all( [oper_id, agent_id], index=OperationPerAgentIndexes.OperationIdAndAgentId).update({ OperationPerAgentKey.CompletedTime: self.db_time }).run(conn)) conn.close() except Exception as e: logger.exception(e)
def update_supported_and_agent_apps(json_data, table=SupportedAppsCollection): if table == SupportedAppsCollection: CurrentAppsKey = SupportedAppsKey LatestDownloadedCollection = LatestDownloadedSupportedCollection AppType = 'supported_apps' elif table == AgentAppsCollection: CurrentAppsKey = AgentAppsKey LatestDownloadedCollection = LatestDownloadedAgentCollection AppType = 'agent_apps' try: rv_q = Queue('downloader', connection=rq_pkg_pool) conn = db_connect() inserted_count = 0 all_customers = (list( r.table(Collection.Customers).map( lambda x: x[CustomerKey.CustomerName]).run(conn))) for i in range(len(json_data)): json_data[i][CurrentAppsKey.Customers] = all_customers json_data[i][CurrentAppsKey.ReleaseDate] = r.epoch_time( json_data[i][CurrentAppsKey.ReleaseDate]) json_data[i][ CurrentAppsKey. FilesDownloadStatus] = PackageCodes.FilePendingDownload json_data[i][CurrentAppsKey.Hidden] = 'no' insert_data_into_table(json_data[i], LatestDownloadedCollection) file_data = json_data[i].get(CurrentAppsKey.FileData) insert_file_data(json_data[i][CurrentAppsKey.AppId], file_data) data_to_update = ({CurrentAppsKey.Customers: all_customers}) exists = (r.table(table).get( json_data[i][CurrentAppsKey.AppId]).run(conn)) if exists: updated = (r.table(table).get(json_data[i][ CurrentAppsKey.AppId]).update(data_to_update).run(conn)) else: updated = (r.table(table).insert(json_data[i]).run(conn)) rv_q.enqueue_call(func=download_all_files_in_app, args=(json_data[i][CurrentAppsKey.AppId], json_data[i][CurrentAppsKey.OsCode], None, file_data, 0, AppType), timeout=86400) inserted_count += updated['inserted'] conn.close() update_apps = IncomingSupportedOrAgentApps(table=table) update_apps.sync_supported_updates_to_all_agents(json_data) except Exception as e: logger.exception(e)
def update_file_data(app_id, agent_id, file_data): conn = db_connect() try: if len(file_data) > 0: for uri in file_data: exists = ( r .table(FilesCollection) .get(uri[FilesKey.FileName]) .run(conn) ) if exists: ( r .table(FilesCollection) .get(uri[FilesKey.FileName]) .update( { FilesKey.AppIds: ( r.row[FilesKey.AppIds] .set_insert(app_id) ), FilesKey.AgentIds: ( r.row[FilesKey.AgentIds] .set_insert(agent_id) ) } ) .run(conn) ) else: ( r .table(FilesCollection) .insert( { FilesKey.AppIds: [app_id], FilesKey.AgentIds: [agent_id], FilesKey.FileName: uri[FilesKey.FileName], FilesKey.FileSize: uri[FilesKey.FileSize], FilesKey.FileUri: uri[FilesKey.FileUri], FilesKey.FileHash: uri[FilesKey.FileHash], }, ) .run(conn, no_reply=True) ) except Exception as e: logger.exception(e) conn.close()
def mouse_exists(mouse_name): conn = db_connect() try: exist = (r.table(RelayServersCollection).get(mouse_name).run(conn)) except Exception as e: logger.exception(e) exist = False conn.close() return (exist)
def oper_with_appid_exists(oper_id, agent_id, app_id): conn = db_connect() exists = None try: exists = (r.table(OperationsPerAppCollection).get_all( [oper_id, agent_id, app_id], index=OperationPerAppIndexes.OperationIdAndAgentIdAndAppId).pluck( OperationPerAppKey.AppId).run(conn)) conn.close() except Exception as e: logger.exception(e) return (exists)
def all_agent_status(): seconds = mktime(datetime.now().timetuple()) - 600 try: conn = db_connect() (r.table(AgentsCollection).filter(lambda x: x[AgentKey.LastAgentUpdate] .to_epoch_time() < seconds).update({ AgentKey.AgentStatus: 'down' }).run(conn)) conn.close() except Exception as e: logger.exception(e)
def update_agent_status(agentid, username, uri=None, method=None): try: conn = db_connect() update_status = { AgentKey.LastAgentUpdate: r.epoch_time( mktime( datetime.now() .timetuple() ) ), AgentKey.AgentStatus: 'up' } exists = ( r .table(AgentsCollection) .get(agentid) .run(conn) ) if exists: ( r .table(AgentsCollection) .get(agentid) .update(update_status) .run(conn) ) status = ( GenericResults( username, uri, method ).object_updated(agentid, 'agent', update_status) ) else: status = ( GenericResults( username, uri, method ).does_not_exists(agentid, 'agent') ) logger.info(status['message']) conn.close() except Exception as e: status = ( GenericResults( username, uri, method ).something_broke(agentid, 'agent', e) ) logger.exception(e)
def sync_supported_updates_to_all_agents(self, apps): try: conn = db_connect() deleted_count = 0 ( r .table(self.CurrentAppsPerAgentCollection) .delete() .run(conn) ) conn.close() self.update_agents_with_supported(apps) except Exception as e: logger.exception(e)
def get_mouse_addresses(customer_name): conn = db_connect() try: exist = list( r.table(RelayServersCollection).pluck( RelayServers.Address).run(conn)) except Exception as e: logger.exception(e) exist = False conn.close() return (exist)
def _update_completed_time_on_agents(self, oper_id, agent_id): try: conn = db_connect() ( r .table(OperationsPerAgentCollection) .get_all( [oper_id, agent_id], index=OperationPerAgentIndexes.OperationIdAndAgentId ) .update({OperationPerAgentKey.CompletedTime: self.db_time}) .run(conn) ) conn.close() except Exception as e: logger.exception(e)
def delete_mouse(mouse_name, username, uri, method): conn = db_connect() try: (r.table(RelayServersCollection).get(mouse_name).delete().run(conn)) status = (MightyMouseResults(username, uri, method).remove(mouse_name)) except Exception as e: status = (MightyMouseResults(username, uri, method).failed_to_remove(mouse_name, e)) logger.exception(status) conn.close() return (status)
def all_agent_status(): seconds = mktime(datetime.now().timetuple()) - 600 try: conn = db_connect() ( r .table(AgentsCollection) .filter( lambda x: x[AgentKey.LastAgentUpdate].to_epoch_time() < seconds ) .update({AgentKey.AgentStatus: 'down'}) .run(conn) ) conn.close() except Exception as e: logger.exception(e)
def get_mouse_addresses(customer_name): conn = db_connect() try: exist = list( r .table(RelayServersCollection) .pluck(RelayServers.Address) .run(conn) ) except Exception as e: logger.exception(e) exist = False conn.close() return(exist)
def mouse_exists(mouse_name): conn = db_connect() try: exist = ( r .table(RelayServersCollection) .get(mouse_name) .run(conn) ) except Exception as e: logger.exception(e) exist = False conn.close() return(exist)
def get_vulnerability_categories(cve_id): conn = db_connect() info = [] try: info = (r.table(CveCollection).get(cve_id).pluck( CveKey.CveCategories).run(conn)) if info: info = info[CveKey.CveCategories] logger.debug('retrieved vulnerabilty categories for cve_id: %s', cve_id) except Exception as e: logger.exception(e) conn.close() return (info)
def oper_with_appid_exists(oper_id, agent_id, app_id): conn = db_connect() exists = None try: exists = ( r .table(OperationsPerAppCollection) .get_all( [oper_id, agent_id, app_id], index=OperationPerAppIndexes.OperationIdAndAgentIdAndAppId ) .pluck(OperationPerAppKey.AppId) .run(conn) ) conn.close() except Exception as e: logger.exception(e) return(exists)
def update_agents_with_supported(self, apps, agents=None): try: conn = db_connect() if agents: for agent in agents: ( r .table(self.CurrentAppsPerAgentCollection) .get_all( agent[self.CurrentAppsPerAgentKey.AgentId], index=self.CurrentAppsPerAgentIndexes.AgentId ) .delete() .run(conn) ) for app in apps: if not agents: agents = get_agents_info(agent_os=app[AgentKey.OsCode]) file_data = app.get(self.CurrentAppsKey.FileData) if agents: for agent in agents: if agent[AgentKey.OsCode] == app[AgentKey.OsCode]: agent[self.CurrentAppsPerAgentKey.AppId] = app[self.CurrentAppsPerAgentKey.AppId] update_file_data( agent[self.CurrentAppsPerAgentKey.AppId], agent[self.CurrentAppsPerAgentKey.AgentId], file_data ) app_per_agent_props = self._set_app_per_agent_properties(**agent) agent_has_app = self.check_if_agent_has_app(agent) if not agent_has_app: self.insert_app( app_per_agent_props ) elif agent_has_app: app_per_agent_props[self.CurrentAppsPerAgentKey.Status] = INSTALLED self.insert_app(app_per_agent_props) conn.close() except Exception as e: logger.exception(e)
def _update_operation_status_code(self, operation_id): try: conn = db_connect() operation = ( r.table(OperationsCollection).get(operation_id).run(conn)) if (operation[OperationKey.AgentsTotalCount] == operation[ OperationKey.AgentsCompletedCount]): (r.table(OperationsCollection).get(operation_id).update({ OperationKey.OperationStatus: OperationCodes.ResultsCompleted, OperationKey.CompletedTime: self.db_time }).run(conn)) elif (operation[OperationKey.AgentsTotalCount] == operation[ OperationKey.AgentsFailedCount]): (r.table(OperationsCollection).get(operation_id).update({ OperationKey.OperationStatus: OperationCodes.ResultsCompletedFailed, OperationKey.CompletedTime: self.db_time }).run(conn)) elif (operation[OperationKey.AgentsTotalCount] == ( operation[OperationKey.AgentsFailedCount] + operation[OperationKey.AgentsCompletedWithErrorsCount])): (r.table(OperationsCollection).get(operation_id).update({ OperationKey.OperationStatus: OperationCodes.ResultsCompletedWithErrors, OperationKey.CompletedTime: self.db_time }).run(conn)) else: (r.table(OperationsCollection).get(operation_id).update({ OperationKey.OperationStatus: OperationCodes.ResultsIncomplete }).run(conn)) conn.close() except Exception as e: logger.exception(e)
def update_customers_in_app(customer_name, app_id, table=AppsCollection): conn = db_connect() if table == AppsCollection: CurrentAppsKey = AppsKey elif table == CustomAppsCollection: CurrentAppsKey = CustomAppsKey elif table == SupportedAppsCollection: CurrentAppsKey = SupportedAppsKey elif table == AgentAppsCollection: CurrentAppsKey = AgentAppsKey try: exists = ( r .table(table) .get(app_id) .run(conn) ) if exists: ( r .table(table) .get(app_id) .update( { CurrentAppsKey.Customers: ( r.row[CurrentAppsKey.Customers] .set_insert(customer_name) ), } ) .run(conn) ) except Exception as e: logger.exception(e) conn.close()
def get_all_mouseys(username, uri, method, customer_name=None): conn = db_connect() try: if not customer_name: data = list(r.table(RelayServersCollection).run(conn)) else: data = list( r.table(RelayServersCollection).filter(lambda x: x[ RelayServers.Customers].contains(customer_name)).run(conn)) status = (GenericResults(username, uri, method).information_retrieved( data, len(data))) except Exception as e: status = (GenericResults(username, uri, method).something_broke( 'retreiving Relay Servers', 'RelayServers', e)) conn.close() return (status)
def add_mouse(customer_names, mouse_name, address, username, uri, method): conn = db_connect() data = { RelayServers.RelayName: mouse_name, RelayServers.Customers: customer_names, RelayServers.Address: address, } try: (r.table(RelayServersCollection).insert(data).run(conn)) status = (MightyMouseResults(username, uri, method).created(mouse_name, data)) except Exception as e: status = (MightyMouseResults(username, uri, method).failed_to_create(mouse_name, e)) logger.exception(status) conn.close() return (status)
def get_vulnerability_categories(cve_id): conn = db_connect() info = [] try: info = ( r .table(CveCollection) .get(cve_id) .pluck(CveKey.CveCategories) .run(conn) ) if info: info = info[CveKey.CveCategories] logger.debug('retrieved vulnerabilty categories for cve_id: %s', cve_id) except Exception as e: logger.exception(e) conn.close() return(info)
def get_ubuntu_cveids(app_name, app_version): conn = db_connect() info = { UbuntuSecurityBulletinKey.BulletinId: '', UbuntuSecurityBulletinKey.CveIds: [] } try: info = list( r.table(UbuntuSecurityBulletinCollection).get_all( [app_name, app_version], index=UbuntuSecurityBulletinIndexes.NameAndVersion).pluck( UbuntuSecurityBulletinKey.BulletinId, UbuntuSecurityBulletinKey.CveIds).run(conn)) if info: if len(info) > 1: info = info[0] logger.debug('retrieved ubuntu bulletin info: %s', info) except Exception as e: logger.exception(e) return (info)
def update_agents_with_supported(self, apps, agents=None): try: conn = db_connect() if agents: for agent in agents: (r.table(self.CurrentAppsPerAgentCollection).get_all( agent[self.CurrentAppsPerAgentKey.AgentId], index=self.CurrentAppsPerAgentIndexes.AgentId).delete( ).run(conn)) for app in apps: if not agents: agents = get_agents_info(agent_os=app[AgentKey.OsCode]) file_data = app.get(self.CurrentAppsKey.FileData) if agents: for agent in agents: if agent[AgentKey.OsCode] == app[AgentKey.OsCode]: agent[self.CurrentAppsPerAgentKey.AppId] = app[ self.CurrentAppsPerAgentKey.AppId] update_file_data( agent[self.CurrentAppsPerAgentKey.AppId], agent[self.CurrentAppsPerAgentKey.AgentId], file_data) app_per_agent_props = self._set_app_per_agent_properties( **agent) agent_has_app = self.check_if_agent_has_app(agent) if not agent_has_app: self.insert_app(app_per_agent_props) elif agent_has_app: app_per_agent_props[self.CurrentAppsPerAgentKey .Status] = INSTALLED self.insert_app(app_per_agent_props) conn.close() except Exception as e: logger.exception(e)
def get_file_data(app_id, agent_id=None): conn = db_connect() try: if agent_id: file_data = list( r .table(FilesCollection) .filter( lambda x: ( x[FilesKey.AppIds].contains(app_id) & x[FilesKey.AgentIds].contains(agent_id) ) ) .without(FilesKey.AppIds, FilesKey.AgentIds,) .run(conn) ) else: file_data = list( r .table(FilesCollection) .filter( lambda x: ( x[FilesKey.AppIds].contains(app_id) ) ) .without(FilesKey.AppIds, FilesKey.AgentIds,) .run(conn) ) except Exception as e: file_data = [] logger.exception(e) conn.close() return(file_data)
def get_windows_bulletinid_and_cveids(kb): conn = db_connect() info = { WindowsSecurityBulletinKey.BulletinId: '', WindowsSecurityBulletinKey.CveIds: [] } try: info = list( r.table(WindowsSecurityBulletinCollection).get_all( kb, index=WindowsSecurityBulletinIndexes.ComponentKb).pluck( WindowsSecurityBulletinKey.BulletinId, WindowsSecurityBulletinKey.CveIds).run(conn)) if info: if len(info) > 0: info = info[0] logger.debug('retrieved microsoft bulletin info: %s', info) except Exception as e: logger.exception(e) conn.close() return (info)
def get_all_mouseys(username, uri, method, customer_name=None): conn = db_connect() try: if not customer_name: data = list( r .table(RelayServersCollection) .run(conn) ) else: data = list( r .table(RelayServersCollection) .filter( lambda x: x[RelayServers.Customers].contains(customer_name) ) .run(conn) ) status = ( GenericResults( username, uri, method ).information_retrieved(data, len(data)) ) except Exception as e: status = ( GenericResults( username, uri, method ).something_broke('retreiving Relay Servers', 'RelayServers', e) ) conn.close() return(status)
def update_mouse(exist, mouse_name, customer_names, address, username, uri, method): conn = db_connect() if not address: address = exist[RelayServers.Address] data = { RelayServers.Customers: customer_names, RelayServers.Address: address, } try: a = (r.table(RelayServersCollection).get(mouse_name).update(data).run( conn)) status = (MightyMouseResults(username, uri, method).updated(mouse_name, data)) except Exception as e: status = (MightyMouseResults(username, uri, method).failed_to_update(mouse_name, e)) logger.exception(status) conn.close() return (status)
def initialize_indexes_and_create_tables(): tables = [ ('acls', Id), (AgentsCollection, AgentKey.AgentId), (RelayServersCollection, RelayServers.RelayName), (AppsPerAgentCollection, Id), (CustomAppsCollection, CustomAppsKey.AppId), (CustomAppsPerAgentCollection, Id), (AgentAppsCollection, AgentAppsKey.AppId), (AgentAppsPerAgentCollection, Id), (CveCollection, CveKey.CveId), (WindowsSecurityBulletinCollection, WindowsSecurityBulletinKey.Id), (UbuntuSecurityBulletinCollection, UbuntuSecurityBulletinKey.Id), ('downloaded_status', Id), (FilesCollection, FilesKey.FileName), (HardwarePerAgentCollection, Id), (NotificationCollections.NotificationPlugins, Id), (NotificationCollections.Notifications, NotificationKeys.NotificationId), (NotificationCollections.NotificationsHistory, Id), ('notification_queue', Id), (OperationsCollection, OperationKey.OperationId), (OperationsPerAgentCollection, Id), (OperationsPerAppCollection, Id), ('plugin_configurations', 'name'), (SupportedAppsCollection, SupportedAppsKey.AppId), (LatestDownloadedSupportedCollection, SupportedAppsKey.AppId), (LatestDownloadedAgentCollection, SupportedAppsKey.AppId), (SupportedAppsPerAgentCollection, Id), (TagsCollection, TagsKey.TagId), (TagsPerAgentCollection, Id), (AppsCollection, AppsKey.AppId), ] conn = db_connect() list_of_current_tables = r.table_list().run(conn) for table in tables: if table[0] not in list_of_current_tables: r.table_create(table[0], primary_key=table[1]).run(conn) app_list = r.table(AppsPerAgentCollection).index_list().run(conn) unique_app_list = r.table(AppsCollection).index_list().run(conn) downloaded_list = r.table('downloaded_status').index_list().run(conn) cve_list = r.table(CveCollection).index_list().run(conn) windows_bulletin_list = r.table(WindowsSecurityBulletinCollection).index_list().run(conn) ubuntu_bulletin_list = r.table(UbuntuSecurityBulletinCollection).index_list().run(conn) files_list = r.table(FilesCollection).index_list().run(conn) tags_list = r.table(TagsCollection).index_list().run(conn) agents_list = r.table(AgentsCollection).index_list().run(conn) operations_list = r.table(OperationsCollection).index_list().run(conn) operations_per_agent_list = r.table(OperationsPerAgentCollection).index_list().run(conn) operations_per_app_list = r.table(OperationsPerAppCollection).index_list().run(conn) notif_list = r.table(NotificationCollections.Notifications).index_list().run(conn) notif_history_list = r.table(NotificationCollections.NotificationsHistory).index_list().run(conn) hw_per_agent_list = r.table(HardwarePerAgentCollection).index_list().run(conn) tag_per_agent_list = r.table(TagsPerAgentCollection).index_list().run(conn) notif_plugin_list = r.table(NotificationCollections.NotificationPlugins,).index_list().run(conn) custom_app_list = r.table(CustomAppsCollection).index_list().run(conn) custom_app_per_agent_list = r.table(CustomAppsPerAgentCollection).index_list().run(conn) supported_app_list = r.table(SupportedAppsCollection).index_list().run(conn) supported_app_per_agent_list = r.table(SupportedAppsPerAgentCollection).index_list().run(conn) agent_app_list = r.table(AgentAppsCollection).index_list().run(conn) agent_app_per_agent_list = r.table(AgentAppsPerAgentCollection).index_list().run(conn) #################################### AgentsColleciton Indexes ################################################### if not AgentIndexes.CustomerName in agents_list: r.table(AgentsCollection).index_create(AgentIndexes.CustomerName).run(conn) if not AgentIndexes.OsCode in agents_list: r.table(AgentsCollection).index_create(AgentIndexes.OsCode).run(conn) #################################### AppsCollection Indexes ################################################### if not AppsIndexes.RvSeverity in unique_app_list: r.table(AppsCollection).index_create(AppsIndexes.RvSeverity).run(conn) if not AppsIndexes.Name in unique_app_list: r.table(AppsCollection).index_create(AppsIndexes.Name).run(conn) if not AppsIndexes.NameAndVersion in unique_app_list: r.table(AppsCollection).index_create( AppsIndexes.NameAndVersion, lambda x: [ x[AppsKey.Name], x[AppsKey.Version]]).run(conn) if not AppsIndexes.Customers in unique_app_list: r.table(AppsCollection).index_create(AppsIndexes.Customers, multi=True).run(conn) if not AppsIndexes.CustomerAndRvSeverity in unique_app_list: r.table(AppsCollection).index_create( AppsIndexes.CustomerAndRvSeverity, lambda x: [ x[AppsKey.Customers], x[AppsKey.RvSeverity]], multi=True).run(conn) if not AppsIndexes.AppIdAndRvSeverity in unique_app_list: r.table(AppsCollection).index_create( AppsIndexes.AppIdAndRvSeverity, lambda x: [ x[AppsKey.AppId], x[AppsKey.RvSeverity]]).run(conn) # if not AppsIndexes.AppIdAndRvSeverityAndHidden in unique_app_list: # r.table(AppsCollection).index_create( # AppsIndexes.AppIdAndRvSeverityAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.RvSeverity], # x[AppsKey.Hidden]]).run(conn) # if not AppsIndexes.CustomerAndHidden in unique_app_list: # r.table(AppsCollection).index_create( # AppsIndexes.CustomerAndHidden, lambda x: [ # x[AppsKey.Customers], # x[AppsKey.Hidden]], multi=True).run(conn) # if not AppsIndexes.AppIdAndHidden in unique_app_list: # r.table(AppsCollection).index_create( # AppsIndexes.AppIdAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.Hidden]]).run(conn) #################################### FilesColleciton Indexes ################################################### if not FilesIndexes.FilesDownloadStatus in files_list: r.table(FilesCollection).index_create(FilesIndexes.FilesDownloadStatus).run(conn) #################################### AppsPerAgentCollection Indexes ################################################### if not AppsPerAgentIndexes.Status in app_list: r.table(AppsPerAgentCollection).index_create(AppsPerAgentIndexes.Status).run(conn) if not AppsPerAgentIndexes.AgentId in app_list: r.table(AppsPerAgentCollection).index_create(AppsPerAgentIndexes.AgentId).run(conn) if not AppsPerAgentIndexes.AppId in app_list: r.table(AppsPerAgentCollection).index_create(AppsPerAgentIndexes.AppId).run(conn) if not AppsPerAgentIndexes.CustomerName in app_list: r.table(AppsPerAgentCollection).index_create(AppsPerAgentIndexes.CustomerName).run(conn) if not AppsPerAgentIndexes.AgentIdAndAppId in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.AgentIdAndAppId, lambda x: [ x[AppsPerAgentKey.AgentId], x[AppsPerAgentKey.AppId]]).run(conn) if not AppsPerAgentIndexes.AppIdAndCustomer in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.AppIdAndCustomer, lambda x: [ x[AppsPerAgentKey.AppId], x[AppsPerAgentKey.CustomerName]]).run(conn) if not AppsPerAgentIndexes.AppIdAndStatus in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.AppIdAndStatus, lambda x: [ x[AppsPerAgentKey.AppId], x[AppsPerAgentKey.Status]]).run(conn) if not AppsPerAgentIndexes.StatusAndCustomer in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.StatusAndCustomer, lambda x: [ x[AppsPerAgentKey.Status], x[AppsPerAgentKey.CustomerName]]).run(conn) if not AppsPerAgentIndexes.AppIdAndStatusAndCustomer in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.AppIdAndStatusAndCustomer, lambda x: [ x[AppsPerAgentKey.AppId], x[AppsPerAgentKey.Status], x[AppsPerAgentKey.CustomerName]]).run(conn) if not AppsPerAgentIndexes.StatusAndAgentId in app_list: r.table(AppsPerAgentCollection).index_create( AppsPerAgentIndexes.StatusAndAgentId, lambda x: [ x[AppsPerAgentKey.Status], x[AppsPerAgentKey.AgentId]]).run(conn) #################################### TagsCollection Indexes ################################################### if not TagsIndexes.CustomerName in tags_list: r.table(TagsCollection).index_create(TagsIndexes.CustomerName).run(conn) if not TagsIndexes.TagNameAndCustomer in tags_list: r.table(TagsCollection).index_create( TagsIndexes.TagNameAndCustomer, lambda x: [ x[TagsKey.CustomerName], x[TagsKey.TagName]]).run(conn) #################################### TagsPerAgentCollection Indexes ################################################### if not TagsPerAgentIndexes.TagId in tag_per_agent_list: r.table(TagsPerAgentCollection).index_create(TagsPerAgentIndexes.TagId).run(conn) if not TagsPerAgentIndexes.AgentId in tag_per_agent_list: r.table(TagsPerAgentCollection).index_create(TagsPerAgentIndexes.AgentId).run(conn) if not TagsPerAgentIndexes.AgentIdAndTagId in tag_per_agent_list: r.table(TagsPerAgentCollection).index_create( TagsPerAgentIndexes.AgentIdAndTagId, lambda x: [ x[TagsPerAgentKey.AgentId], x[TagsPerAgentKey.TagId]]).run(conn) #################################### CustomAppsCollection Indexes ################################################### if not CustomAppsIndexes.RvSeverity in custom_app_list: r.table(CustomAppsCollection).index_create(CustomAppsIndexes.RvSeverity).run(conn) if not CustomAppsIndexes.Name in custom_app_list: r.table(CustomAppsCollection).index_create(CustomAppsIndexes.Name).run(conn) if not CustomAppsIndexes.NameAndVersion in custom_app_list: r.table(CustomAppsCollection).index_create( CustomAppsIndexes.NameAndVersion, lambda x: [ x[CustomAppsKey.Name], x[CustomAppsKey.Version]]).run(conn) if not CustomAppsIndexes.Customers in custom_app_list: r.table(CustomAppsCollection).index_create(CustomAppsIndexes.Customers, multi=True).run(conn) if not CustomAppsIndexes.CustomerAndRvSeverity in custom_app_list: r.table(CustomAppsCollection).index_create( CustomAppsIndexes.CustomerAndRvSeverity, lambda x: [ x[CustomAppsKey.Customers], x[CustomAppsKey.RvSeverity]], multi=True).run(conn) if not CustomAppsIndexes.AppIdAndRvSeverity in custom_app_list: r.table(CustomAppsCollection).index_create( CustomAppsIndexes.AppIdAndRvSeverity, lambda x: [ x[CustomAppsKey.AppId], x[CustomAppsKey.RvSeverity]]).run(conn) # if not CustomAppsIndexes.AppIdAndRvSeverityAndHidden in custom_app_list: # r.table(CustomAppsCollection).index_create( # CustomAppsIndexes.AppIdAndRvSeverityAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.RvSeverity], # x[AppsKey.Hidden]]).run(conn) # # if not CustomAppsIndexes.CustomerAndHidden in custom_app_list: # r.table(CustomAppsCollection).index_create( # CustomAppsIndexes.CustomerAndHidden, lambda x: [ # x[AppsKey.Customers], # x[AppsKey.Hidden]], multi=True).run(conn) # # if not CustomAppsIndexes.AppIdAndHidden in custom_app_list: # r.table(CustomAppsCollection).index_create( # CustomAppsIndexes.AppIdAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.Hidden]]).run(conn) # #################################### CustomAppsPerAgentCollection Indexes ################################################### if not CustomAppsPerAgentIndexes.Status in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create(CustomAppsPerAgentIndexes.Status).run(conn) if not CustomAppsPerAgentIndexes.AgentId in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create(CustomAppsPerAgentIndexes.AgentId).run(conn) if not CustomAppsPerAgentIndexes.AppId in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create(CustomAppsPerAgentIndexes.AppId).run(conn) if not CustomAppsPerAgentIndexes.CustomerName in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create(CustomAppsPerAgentIndexes.CustomerName).run(conn) if not CustomAppsPerAgentIndexes.AgentIdAndAppId in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.AgentIdAndAppId, lambda x: [ x[CustomAppsPerAgentKey.AgentId], x[CustomAppsPerAgentKey.AppId]]).run(conn) if not CustomAppsPerAgentIndexes.AppIdAndCustomer in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.AppIdAndCustomer, lambda x: [ x[CustomAppsPerAgentKey.AppId], x[CustomAppsPerAgentKey.CustomerName]]).run(conn) if not CustomAppsPerAgentIndexes.AppIdAndStatus in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.AppIdAndStatus, lambda x: [ x[CustomAppsPerAgentKey.AppId], x[CustomAppsPerAgentKey.Status]]).run(conn) if not CustomAppsPerAgentIndexes.StatusAndCustomer in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.StatusAndCustomer, lambda x: [ x[CustomAppsPerAgentKey.Status], x[CustomAppsPerAgentKey.CustomerName]]).run(conn) if not CustomAppsPerAgentIndexes.AppIdAndStatusAndCustomer in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.AppIdAndStatusAndCustomer, lambda x: [ x[CustomAppsPerAgentKey.AppId], x[CustomAppsPerAgentKey.Status], x[CustomAppsPerAgentKey.CustomerName]]).run(conn) if not CustomAppsPerAgentIndexes.StatusAndAgentId in custom_app_per_agent_list: r.table(CustomAppsPerAgentCollection).index_create( CustomAppsPerAgentIndexes.StatusAndAgentId, lambda x: [ x[CustomAppsPerAgentKey.Status], x[CustomAppsPerAgentKey.AgentId]]).run(conn) #################################### SupportedAppsCollection Indexes ################################################### if not SupportedAppsIndexes.RvSeverity in supported_app_list: r.table(SupportedAppsCollection).index_create(SupportedAppsIndexes.RvSeverity).run(conn) if not SupportedAppsIndexes.Name in supported_app_list: r.table(SupportedAppsCollection).index_create(SupportedAppsIndexes.Name).run(conn) if not SupportedAppsIndexes.NameAndVersion in supported_app_list: r.table(SupportedAppsCollection).index_create( SupportedAppsIndexes.NameAndVersion, lambda x: [ x[SupportedAppsKey.Name], x[SupportedAppsKey.Version]]).run(conn) if not SupportedAppsIndexes.Customers in supported_app_list: r.table(SupportedAppsCollection).index_create(SupportedAppsIndexes.Customers, multi=True).run(conn) if not SupportedAppsIndexes.CustomerAndRvSeverity in supported_app_list: r.table(SupportedAppsCollection).index_create( SupportedAppsIndexes.CustomerAndRvSeverity, lambda x: [ x[SupportedAppsKey.Customers], x[SupportedAppsKey.RvSeverity]], multi=True).run(conn) if not SupportedAppsIndexes.AppIdAndRvSeverity in supported_app_list: r.table(SupportedAppsCollection).index_create( SupportedAppsIndexes.AppIdAndRvSeverity, lambda x: [ x[SupportedAppsKey.AppId], x[SupportedAppsKey.RvSeverity]]).run(conn) # if not SupportedAppsIndexes.AppIdAndRvSeverityAndHidden in supported_app_list: # r.table(SupportedAppsCollection).index_create( # SupportedAppsIndexes.AppIdAndRvSeverityAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.RvSeverity], # x[AppsKey.Hidden]]).run(conn) # # if not SupportedAppsIndexes.CustomerAndHidden in supported_app_list: # r.table(SupportedAppsCollection).index_create( # SupportedAppsIndexes.CustomerAndHidden, lambda x: [ # x[AppsKey.Customers], # x[AppsKey.Hidden]], multi=True).run(conn) # # if not SupportedAppsIndexes.AppIdAndHidden in supported_app_list: # r.table(SupportedAppsCollection).index_create( # SupportedAppsIndexes.AppIdAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.Hidden]]).run(conn) #################################### SupportedAppsPerAgentCollection Indexes ################################################### if not SupportedAppsPerAgentIndexes.Status in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create(SupportedAppsPerAgentIndexes.Status).run(conn) if not SupportedAppsPerAgentIndexes.AgentId in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create(SupportedAppsPerAgentIndexes.AgentId).run(conn) if not SupportedAppsPerAgentIndexes.AppId in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create(SupportedAppsPerAgentIndexes.AppId).run(conn) if not SupportedAppsPerAgentIndexes.CustomerName in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create(SupportedAppsPerAgentIndexes.CustomerName).run(conn) if not SupportedAppsPerAgentIndexes.AgentIdAndAppId in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.AgentIdAndAppId, lambda x: [ x[SupportedAppsPerAgentKey.AgentId], x[SupportedAppsPerAgentKey.AppId]]).run(conn) if not SupportedAppsPerAgentIndexes.AppIdAndCustomer in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.AppIdAndCustomer, lambda x: [ x[SupportedAppsPerAgentKey.AppId], x[SupportedAppsPerAgentKey.CustomerName]]).run(conn) if not SupportedAppsPerAgentIndexes.AppIdAndStatus in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.AppIdAndStatus, lambda x: [ x[SupportedAppsPerAgentKey.AppId], x[SupportedAppsPerAgentKey.Status]]).run(conn) if not SupportedAppsPerAgentIndexes.StatusAndCustomer in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.StatusAndCustomer, lambda x: [ x[SupportedAppsPerAgentKey.Status], x[SupportedAppsPerAgentKey.CustomerName]]).run(conn) if not SupportedAppsPerAgentIndexes.AppIdAndStatusAndCustomer in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.AppIdAndStatusAndCustomer, lambda x: [ x[SupportedAppsPerAgentKey.AppId], x[SupportedAppsPerAgentKey.Status], x[SupportedAppsPerAgentKey.CustomerName]]).run(conn) if not SupportedAppsPerAgentIndexes.StatusAndAgentId in supported_app_per_agent_list: r.table(SupportedAppsPerAgentCollection).index_create( SupportedAppsPerAgentIndexes.StatusAndAgentId, lambda x: [ x[SupportedAppsPerAgentKey.Status], x[SupportedAppsPerAgentKey.AgentId]]).run(conn) #################################### AgentAppsCollection Indexes ################################################### if not AgentAppsIndexes.RvSeverity in agent_app_list: r.table(AgentAppsCollection).index_create(AgentAppsIndexes.RvSeverity).run(conn) if not AgentAppsIndexes.Name in agent_app_list: r.table(AgentAppsCollection).index_create(AgentAppsIndexes.Name).run(conn) if not AgentAppsIndexes.NameAndVersion in agent_app_list: r.table(AgentAppsCollection).index_create( AgentAppsIndexes.NameAndVersion, lambda x: [ x[AgentAppsKey.Name], x[AgentAppsKey.Version]]).run(conn) if not AgentAppsIndexes.Customers in agent_app_list: r.table(AgentAppsCollection).index_create(AgentAppsIndexes.Customers, multi=True).run(conn) if not AgentAppsIndexes.CustomerAndRvSeverity in agent_app_list: r.table(AgentAppsCollection).index_create( AgentAppsIndexes.CustomerAndRvSeverity, lambda x: [ x[AgentAppsKey.Customers], x[AgentAppsKey.RvSeverity]], multi=True).run(conn) if not AgentAppsIndexes.AppIdAndRvSeverity in agent_app_list: r.table(AgentAppsCollection).index_create( AgentAppsIndexes.AppIdAndRvSeverity, lambda x: [ x[AgentAppsKey.AppId], x[AgentAppsKey.RvSeverity]]).run(conn) # if not AgentAppsIndexes.AppIdAndRvSeverityAndHidden in agent_app_list: # r.table(AgentAppsCollection).index_create( # AgentAppsIndexes.AppIdAndRvSeverityAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.RvSeverity], # x[AppsKey.Hidden]]).run(conn) # # if not AgentAppsIndexes.CustomerAndHidden in agent_app_list: # r.table(AgentAppsCollection).index_create( # AgentAppsIndexes.CustomerAndHidden, lambda x: [ # x[AppsKey.Customers], # x[AppsKey.Hidden]], multi=True).run(conn) # # if not AgentAppsIndexes.AppIdAndHidden in agent_app_list: # r.table(AgentAppsCollection).index_create( # AgentAppsIndexes.AppIdAndHidden, lambda x: [ # x[AppsKey.AppId], # x[AppsKey.Hidden]]).run(conn) #################################### AgentAppsPerAgentCollection Indexes ################################################### if not AgentAppsPerAgentIndexes.Status in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create(AgentAppsPerAgentIndexes.Status).run(conn) if not AgentAppsPerAgentIndexes.AgentId in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create(AgentAppsPerAgentIndexes.AgentId).run(conn) if not AgentAppsPerAgentIndexes.AppId in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create(AgentAppsPerAgentIndexes.AppId).run(conn) if not AgentAppsPerAgentIndexes.CustomerName in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create(AgentAppsPerAgentIndexes.CustomerName).run(conn) if not AgentAppsPerAgentIndexes.AgentIdAndAppId in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.AgentIdAndAppId, lambda x: [ x[AgentAppsPerAgentKey.AgentId], x[AgentAppsPerAgentKey.AppId]]).run(conn) if not AgentAppsPerAgentIndexes.AppIdAndCustomer in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.AppIdAndCustomer, lambda x: [ x[AgentAppsPerAgentKey.AppId], x[AgentAppsPerAgentKey.CustomerName]]).run(conn) if not AgentAppsPerAgentIndexes.AppIdAndStatus in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.AppIdAndStatus, lambda x: [ x[AgentAppsPerAgentKey.AppId], x[AgentAppsPerAgentKey.Status]]).run(conn) if not AgentAppsPerAgentIndexes.StatusAndCustomer in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.StatusAndCustomer, lambda x: [ x[AgentAppsPerAgentKey.Status], x[AgentAppsPerAgentKey.CustomerName]]).run(conn) if not AgentAppsPerAgentIndexes.AppIdAndStatusAndCustomer in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.AppIdAndStatusAndCustomer, lambda x: [ x[AgentAppsPerAgentKey.AppId], x[AgentAppsPerAgentKey.Status], x[AgentAppsPerAgentKey.CustomerName]]).run(conn) if not AgentAppsPerAgentIndexes.StatusAndAgentId in agent_app_per_agent_list: r.table(AgentAppsPerAgentCollection).index_create( AgentAppsPerAgentIndexes.StatusAndAgentId, lambda x: [ x[AgentAppsPerAgentKey.Status], x[AgentAppsPerAgentKey.AgentId]]).run(conn) #################################### OperationsCollection Indexes ################################################### if not OperationIndexes.CustomerName in operations_list: r.table(OperationsCollection).index_create(OperationKey.CustomerName).run(conn) if not OperationIndexes.TagId in operations_list: r.table(OperationsCollection).index_create(OperationKey.TagId).run(conn) if not OperationIndexes.Operation in operations_list: r.table(OperationsCollection).index_create(OperationKey.Operation).run(conn) if not OperationIndexes.OperationAndCustomer in operations_list: r.table(OperationsCollection).index_create( OperationIndexes.OperationAndCustomer, lambda x: [ x[OperationKey.Operation], x[OperationKey.CustomerName]]).run(conn) if not OperationIndexes.PluginAndCustomer in operations_list: r.table(OperationsCollection).index_create( OperationIndexes.PluginAndCustomer, lambda x: [ x[OperationKey.Plugin], x[OperationKey.CustomerName]]).run(conn) if not OperationIndexes.CreatedByAndCustomer in operations_list: r.table(OperationsCollection).index_create( OperationIndexes.CreatedByAndCustomer, lambda x: [ x[OperationKey.CreatedBy], x[OperationKey.CustomerName]]).run(conn) #################################### OperationsPerAgentCollection Indexes ################################################### if not OperationPerAgentIndexes.OperationId in operations_per_agent_list: r.table(OperationsPerAgentCollection).index_create(OperationPerAgentKey.OperationId).run(conn) if not OperationPerAgentIndexes.AgentIdAndCustomer in operations_per_agent_list: r.table(OperationsPerAgentCollection).index_create( OperationPerAgentIndexes.AgentIdAndCustomer, lambda x: [ x[OperationPerAgentKey.AgentId], x[OperationPerAgentKey.CustomerName]]).run(conn) if not OperationPerAgentIndexes.TagIdAndCustomer in operations_per_agent_list: r.table(OperationsPerAgentCollection).index_create( OperationPerAgentIndexes.TagIdAndCustomer, lambda x: [ x[OperationPerAgentKey.TagId], x[OperationPerAgentKey.CustomerName]]).run(conn) if not OperationPerAgentIndexes.StatusAndCustomer in operations_per_agent_list: r.table(OperationsPerAgentCollection).index_create( OperationPerAgentIndexes.StatusAndCustomer, lambda x: [ x[OperationPerAgentKey.Status], x[OperationPerAgentKey.CustomerName]]).run(conn) if not OperationPerAgentIndexes.OperationIdAndAgentId in operations_per_agent_list: r.table(OperationsPerAgentCollection).index_create( OperationPerAgentIndexes.OperationIdAndAgentId, lambda x: [ x[OperationPerAgentKey.OperationId], x[OperationPerAgentKey.AgentId]]).run(conn) #################################### OperationsPerAppCollection Indexes ################################################### if not OperationPerAppIndexes.OperationId in operations_per_app_list: r.table(OperationsPerAppCollection).index_create(OperationPerAppKey.OperationId).run(conn) if not OperationPerAppIndexes.OperationIdAndAgentId in operations_per_app_list: r.table(OperationsPerAppCollection).index_create( OperationPerAppIndexes.OperationIdAndAgentId, lambda x: [ x[OperationPerAppKey.OperationId], x[OperationPerAppKey.AgentId]]).run(conn) if not OperationPerAppIndexes.OperationIdAndAgentIdAndAppId in operations_per_app_list: r.table(OperationsPerAppCollection).index_create( OperationPerAppIndexes.OperationIdAndAgentIdAndAppId, lambda x: [ x[OperationPerAppKey.OperationId], x[OperationPerAppKey.AgentId], x[OperationPerAppKey.AppId]]).run(conn) #################################### HardwarePerAgentCollection Indexes ################################################### if not HardwarePerAgentIndexes.Type in hw_per_agent_list: r.table(HardwarePerAgentCollection).index_create(HardwarePerAgentIndexes.Type).run(conn) if not HardwarePerAgentIndexes.AgentId in hw_per_agent_list: r.table(HardwarePerAgentCollection).index_create(HardwarePerAgentIndexes.AgentId).run(conn) #################################### DownloadStatusCollection Indexes ################################################### if not 'app_id' in downloaded_list: r.table('downloaded_status').index_create('app_id').run(conn) if not 'by_filename_and_rvid' in downloaded_list: r.table('downloaded_status').index_create( 'by_filename_and_rvid', lambda x: [ x['file_name'], x['app_id']]).run(conn) #################################### NotificationsCollection Indexes ################################################### if not NotificationIndexes.CustomerName in notif_list: r.table(NotificationCollections.Notifications).index_create(NotificationKeys.CustomerName).run(conn) if not NotificationIndexes.RuleNameAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.RuleNameAndCustomer, lambda x: [ x[NotificationKeys.RuleName], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.NotificationTypeAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.NotificationTypeAndCustomer, lambda x: [ x[NotificationKeys.NotificationType], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.AppThresholdAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.AppThresholdAndCustomer, lambda x: [ x[NotificationKeys.AppThreshold], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.RebootThresholdAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.RebootThresholdAndCustomer, lambda x: [ x[NotificationKeys.RebootThreshold], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.ShutdownThresholdAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.ShutdownThresholdAndCustomer, lambda x: [ x[NotificationKeys.ShutdownThreshold], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.CpuThresholdAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.CpuThresholdAndCustomer, lambda x: [ x[NotificationKeys.CpuThreshold], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.MemThresholdAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.MemThresholdAndCustomer, lambda x: [ x[NotificationKeys.MemThreshold], x[NotificationKeys.CustomerName]]).run(conn) if not NotificationIndexes.FileSystemThresholdAndFileSystemAndCustomer in notif_list: r.table(NotificationCollections.Notifications).index_create( NotificationIndexes.FileSystemThresholdAndFileSystemAndCustomer, lambda x: [ x[NotificationKeys.FileSystemThreshold], x[NotificationKeys.FileSystem], x[NotificationKeys.CustomerName]]).run(conn) #################################### NotificationsHistory Indexes ################################################### if not NotificationHistoryIndexes.NotificationId in notif_history_list: r.table(NotificationCollections.NotificationsHistory).index_create(NotificationHistoryKeys.NotificationId).run(conn) #################################### NotificationsPlugin Indexes ################################################### if not NotificationPluginIndexes.CustomerName in notif_plugin_list: r.table(NotificationCollections.NotificationPlugins).index_create(NotificationPluginKeys.CustomerName).run(conn) #################################### Cve Indexes ################################################### if not CveIndexes.CveCategories in cve_list: r.table(CveCollection).index_create(CveIndexes.CveCategories, multi=True).run(conn) #################################### Windows Bulletin Indexes ################################################### if not WindowsSecurityBulletinIndexes.BulletinId in windows_bulletin_list: r.table(WindowsSecurityBulletinCollection).index_create(WindowsSecurityBulletinIndexes.BulletinId).run(conn) if not WindowsSecurityBulletinIndexes.ComponentKb in windows_bulletin_list: r.table(WindowsSecurityBulletinCollection).index_create(WindowsSecurityBulletinIndexes.ComponentKb).run(conn) if not WindowsSecurityBulletinIndexes.CveIds in windows_bulletin_list: r.table(WindowsSecurityBulletinCollection).index_create(WindowsSecurityBulletinIndexes.CveIds, multi=True).run(conn) #################################### Ubuntu Bulletin Indexes ################################################### if not UbuntuSecurityBulletinIndexes.BulletinId in ubuntu_bulletin_list: r.table(UbuntuSecurityBulletinCollection).index_create(UbuntuSecurityBulletinIndexes.BulletinId).run(conn) if not UbuntuSecurityBulletinIndexes.NameAndVersion in ubuntu_bulletin_list: r.table(UbuntuSecurityBulletinCollection).index_create( UbuntuSecurityBulletinIndexes.NameAndVersion, lambda x: x[UbuntuSecurityBulletinKey.Apps].map(lambda y: [y['name'], y['version']]), multi=True).run(conn) conn.close()
rql_msg = 'Rethink couldnt be stopped\n' try: shutil.rmtree(RETHINK_INSTANCES_PATH) msg = 'Rethink instances.d directory removed and cleaned' except Exception as e: msg = 'Rethink instances.d directory could not be removed' completed = False if rql_msg and msg: msg = rql_msg + msg elif rql_msg and not msg: msg = rql_msg return completed, msg if __name__ == '__main__': conn = db_connect() if conn: connected = True rql_msg = 'Rethink is Running' else: connected = False rql_msg = 'Rethink is not Running' print rql_msg db_clean, db_msg = clean_database(connected) print db_msg db_initialized, msg = initialize_db() initialized = False if db_initialized: print 'vFense environment has been succesfully initialized\n' subprocess.Popen( ['chown', '-R', 'toppatch.toppatch', '/opt/TopPatch'], )
def initialize_db(): os.umask(0) if not os.path.exists('/opt/TopPatch/var/tmp'): os.mkdir('/opt/TopPatch/var/tmp') if not os.path.exists('/opt/TopPatch/var/log'): os.mkdir('/opt/TopPatch/var/log') if not os.path.exists('/opt/TopPatch/var/rethinkdb'): os.mkdir('/opt/TopPatch/var/rethinkdb') if not os.path.exists('/opt/TopPatch/var/scheduler'): os.mkdir('/opt/TopPatch/var/scheduler') if not os.path.exists('/opt/TopPatch/var/packages'): os.mkdir('/opt/TopPatch/var/packages') if not os.path.exists('/opt/TopPatch/logs'): os.mkdir('/opt/TopPatch/logs') if not os.path.exists('/opt/TopPatch/var/packages/tmp'): os.mkdir('/opt/TopPatch/var/packages/tmp', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/xls'): os.makedirs('/opt/TopPatch/tp/src/plugins/cve/data/xls', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/xml'): os.mkdir('/opt/TopPatch/tp/src/plugins/cve/data/xml', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/html/ubuntu'): os.makedirs('/opt/TopPatch/tp/src/plugins/cve/data/html/ubuntu', 0773) if not os.path.exists('/usr/lib/libpcre.so.1'): os.symlink('/opt/TopPatch/lib/libpcre.so.1', '/usr/lib') if not os.path.exists('/etc/init.d/vFense'): subprocess.Popen([ 'ln', '-s', '/opt/TopPatch/tp/src/daemon/vFense', '/etc/init.d/vFense' ], ) subprocess.Popen(['update-rc.d', 'vFense', 'defaults'], ) if not os.path.exists('/etc/init.d/nginx'): subprocess.Popen([ 'ln', '-s', '/opt/TopPatch/tp/src/daemon/nginx', '/etc/init.d/nginx' ], ) subprocess.Popen(['update-rc.d', 'nginx', 'defaults'], ) try: tp_exists = pwd.getpwnam('toppatch') except Exception as e: subprocess.Popen([ 'adduser', 'toppatch', ], ) os.chdir(RETHINK_PATH) rethink_init = subprocess.Popen( ['./rethinkdb', 'create', '-d', RETHINK_INSTANCES_PATH], stdout=subprocess.PIPE) rethink_init.poll() rethink_init.wait() if rethink_init.returncode == 0: rethink_start = subprocess.Popen([ './rethinkdb', '--config-file', RETHINK_CONF, '--web-static-directory', RETHINK_WEB ]) rethink_start.poll() completed = True sleep(2) while not db_connect(): print 'Sleeping until rethink starts' sleep(2) else: completed = False msg = 'Failed during Rethink initialization' return (completed, msg) if completed: conn = r.connect(port=9009) r.db_create('toppatch_server').run(conn) db = r.db('toppatch_server') conn.close() ci.initialize_indexes_and_create_tables() conn = db_connect() hierarchy_db.init() Hierarchy.create_customer( DefaultCustomer, { CoreProperty.NetThrottle: '0', CoreProperty.CpuThrottle: 'idle', CoreProperty.PackageUrl: url }) admin_pass = args.admin_password Hierarchy.create_user('admin', 'TopPatch Admin Account', '*****@*****.**', admin_pass, groups=[DefaultGroup.Administrator]) if args.cve_data: print "Updating CVE's..." load_up_all_xml_into_db() print "Done Updating CVE's..." print "Updating Microsoft Security Bulletin Ids..." parse_bulletin_and_updatedb() print "Done Updating Microsoft Security Bulletin Ids..." print "Updating Ubuntu Security Bulletin Ids...( This can take a couple of minutes )" begin_usn_home_page_processing(full_parse=True) print "Done Updating Ubuntu Security Bulletin Ids..." print 'Admin user and password = admin:%s' % (admin_pass) agent_pass = generate_pass() agent = Hierarchy.create_user('agent', 'TopPatch Agent Communication Account', '*****@*****.**', agent_pass, groups=[DefaultGroup.Administrator]) print 'Agent user and password = agent:%s' % (agent_pass) monit.monit_initialization() conn.close() completed = True msg = 'Rethink Initialization and Table creation is now complete' pid = open(RETHINK_PID_FILE, 'r').read() if re.search(r'[0-9]+', pid): try: os.kill(int(pid), signal.SIGTERM) os.remove(RETHINK_PID_FILE) except Exception as e: if e.errno == 3: os.remove(RETHINK_PID_FILE) rql_msg = 'Rethink stopped successfully\n' else: rql_msg = 'Rethink could not be stopped\n' print rql_msg return completed, msg else: completed = False msg = 'Failed during Rethink startup process' return completed, msg
rql_msg = 'Rethink couldnt be stopped\n' try: shutil.rmtree(RETHINK_INSTANCES_PATH) msg = 'Rethink instances.d directory removed and cleaned' except Exception as e: msg = 'Rethink instances.d directory could not be removed' completed = False if rql_msg and msg: msg = rql_msg + msg elif rql_msg and not msg: msg = rql_msg return completed, msg if __name__ == '__main__': conn = db_connect() if conn: connected = True rql_msg = 'Rethink is Running' else: connected = False rql_msg = 'Rethink is not Running' print rql_msg db_clean, db_msg = clean_database(connected) print db_msg db_initialized, msg = initialize_db() initialized = False if db_initialized: print 'vFense environment has been succesfully initialized\n' subprocess.Popen( [
def initialize_db(): os.umask(0) if not os.path.exists('/opt/TopPatch/var/tmp'): os.mkdir('/opt/TopPatch/var/tmp') if not os.path.exists('/opt/TopPatch/var/log'): os.mkdir('/opt/TopPatch/var/log') if not os.path.exists('/opt/TopPatch/var/rethinkdb'): os.mkdir('/opt/TopPatch/var/rethinkdb') if not os.path.exists('/opt/TopPatch/var/scheduler'): os.mkdir('/opt/TopPatch/var/scheduler') if not os.path.exists('/opt/TopPatch/var/packages'): os.mkdir('/opt/TopPatch/var/packages') if not os.path.exists('/opt/TopPatch/logs'): os.mkdir('/opt/TopPatch/logs') if not os.path.exists('/opt/TopPatch/var/packages/tmp'): os.mkdir('/opt/TopPatch/var/packages/tmp', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/xls'): os.makedirs('/opt/TopPatch/tp/src/plugins/cve/data/xls', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/xml'): os.mkdir('/opt/TopPatch/tp/src/plugins/cve/data/xml', 0773) if not os.path.exists('/opt/TopPatch/tp/src/plugins/cve/data/html/ubuntu'): os.makedirs('/opt/TopPatch/tp/src/plugins/cve/data/html/ubuntu', 0773) if not os.path.exists('/usr/lib/libpcre.so.1'): os.symlink('/opt/TopPatch/lib/libpcre.so.1', '/usr/lib') if not os.path.exists('/etc/init.d/vFense'): subprocess.Popen( [ 'ln', '-s', '/opt/TopPatch/tp/src/daemon/vFense', '/etc/init.d/vFense' ], ) subprocess.Popen( [ 'update-rc.d', 'vFense', 'defaults' ], ) if not os.path.exists('/etc/init.d/nginx'): subprocess.Popen( [ 'ln', '-s', '/opt/TopPatch/tp/src/daemon/nginx', '/etc/init.d/nginx' ], ) subprocess.Popen( [ 'update-rc.d', 'nginx', 'defaults' ], ) try: tp_exists = pwd.getpwnam('toppatch') except Exception as e: subprocess.Popen( [ 'adduser', 'toppatch', ], ) os.chdir(RETHINK_PATH) rethink_init = subprocess.Popen(['./rethinkdb', 'create', '-d', RETHINK_INSTANCES_PATH], stdout=subprocess.PIPE) rethink_init.poll() rethink_init.wait() if rethink_init.returncode == 0: rethink_start = subprocess.Popen(['./rethinkdb', '--config-file', RETHINK_CONF, '--web-static-directory', RETHINK_WEB]) rethink_start.poll() completed = True sleep(2) while not db_connect(): print 'Sleeping until rethink starts' sleep(2) else: completed = False msg = 'Failed during Rethink initialization' return(completed, msg) if completed: conn = r.connect(port=9009) r.db_create('toppatch_server').run(conn) db = r.db('toppatch_server') conn.close() ci.initialize_indexes_and_create_tables() conn = db_connect() hierarchy_db.init() Hierarchy.create_customer( DefaultCustomer, { CoreProperty.NetThrottle: '0', CoreProperty.CpuThrottle: 'idle', CoreProperty.PackageUrl: url } ) admin_pass = args.admin_password Hierarchy.create_user( 'admin', 'TopPatch Admin Account', '*****@*****.**', admin_pass, groups=[DefaultGroup.Administrator] ) if args.cve_data: print "Updating CVE's..." load_up_all_xml_into_db() print "Done Updating CVE's..." print "Updating Microsoft Security Bulletin Ids..." parse_bulletin_and_updatedb() print "Done Updating Microsoft Security Bulletin Ids..." print "Updating Ubuntu Security Bulletin Ids...( This can take a couple of minutes )" begin_usn_home_page_processing(full_parse=True) print "Done Updating Ubuntu Security Bulletin Ids..." print 'Admin user and password = admin:%s' % (admin_pass) agent_pass = generate_pass() agent = Hierarchy.create_user( 'agent', 'TopPatch Agent Communication Account', '*****@*****.**', agent_pass, groups=[DefaultGroup.Administrator] ) print 'Agent user and password = agent:%s' % (agent_pass) monit.monit_initialization() conn.close() completed = True msg = 'Rethink Initialization and Table creation is now complete' pid = open(RETHINK_PID_FILE, 'r').read() if re.search(r'[0-9]+', pid): try: os.kill(int(pid), signal.SIGTERM) os.remove(RETHINK_PID_FILE) except Exception as e: if e.errno == 3: os.remove(RETHINK_PID_FILE) rql_msg = 'Rethink stopped successfully\n' else: rql_msg = 'Rethink could not be stopped\n' print rql_msg return completed, msg else: completed = False msg = 'Failed during Rethink startup process' return completed, msg
def update_supported_and_agent_apps(json_data, table=SupportedAppsCollection): if table == SupportedAppsCollection: CurrentAppsKey = SupportedAppsKey LatestDownloadedCollection = LatestDownloadedSupportedCollection AppType = 'supported_apps' elif table == AgentAppsCollection: CurrentAppsKey = AgentAppsKey LatestDownloadedCollection = LatestDownloadedAgentCollection AppType = 'agent_apps' try: rv_q = Queue('downloader', connection=rq_pkg_pool) conn = db_connect() inserted_count = 0 all_customers = ( list( r .table(Collection.Customers) .map(lambda x: x[CustomerKey.CustomerName]) .run(conn) ) ) for i in range(len(json_data)): json_data[i][CurrentAppsKey.Customers] = all_customers json_data[i][CurrentAppsKey.ReleaseDate] = r.epoch_time(json_data[i][CurrentAppsKey.ReleaseDate]) json_data[i][CurrentAppsKey.FilesDownloadStatus] = PackageCodes.FilePendingDownload json_data[i][CurrentAppsKey.Hidden] = 'no' insert_data_into_table(json_data[i], LatestDownloadedCollection) file_data = json_data[i].get(CurrentAppsKey.FileData) insert_file_data(json_data[i][CurrentAppsKey.AppId], file_data) data_to_update = ( { CurrentAppsKey.Customers: all_customers } ) exists = ( r .table(table) .get(json_data[i][CurrentAppsKey.AppId]) .run(conn) ) if exists: updated = ( r .table(table) .get(json_data[i][CurrentAppsKey.AppId]) .update(data_to_update) .run(conn) ) else: updated = ( r .table(table) .insert(json_data[i]) .run(conn) ) rv_q.enqueue_call( func=download_all_files_in_app, args=( json_data[i][CurrentAppsKey.AppId], json_data[i][CurrentAppsKey.OsCode], None, file_data, 0, AppType ), timeout=86400 ) inserted_count += updated['inserted'] conn.close() update_apps = IncomingSupportedOrAgentApps(table=table) update_apps.sync_supported_updates_to_all_agents(json_data) except Exception as e: logger.exception(e)
def _update_operation_status_code(self, operation_id): try: conn = db_connect() operation = ( r .table(OperationsCollection) .get(operation_id) .run(conn) ) if (operation[OperationKey.AgentsTotalCount] == operation[OperationKey.AgentsCompletedCount]): ( r .table(OperationsCollection) .get(operation_id) .update( { OperationKey.OperationStatus: OperationCodes.ResultsCompleted, OperationKey.CompletedTime: self.db_time } ) .run(conn) ) elif (operation[OperationKey.AgentsTotalCount] == operation[OperationKey.AgentsFailedCount]): ( r .table(OperationsCollection) .get(operation_id) .update( { OperationKey.OperationStatus: OperationCodes.ResultsCompletedFailed, OperationKey.CompletedTime: self.db_time } ) .run(conn) ) elif (operation[OperationKey.AgentsTotalCount] == ( operation[OperationKey.AgentsFailedCount] + operation[OperationKey.AgentsCompletedWithErrorsCount] )): ( r .table(OperationsCollection) .get(operation_id) .update( { OperationKey.OperationStatus: OperationCodes.ResultsCompletedWithErrors, OperationKey.CompletedTime: self.db_time } ) .run(conn) ) else: ( r .table(OperationsCollection) .get(operation_id) .update( { OperationKey.OperationStatus: OperationCodes.ResultsIncomplete } ) .run(conn) ) conn.close() except Exception as e: logger.exception(e)