def ingest_to_db(model, session, **kwargs): if not session: from cqlengine import connection LOG.info("No c* session found.. creating one.") connection.setup([config.get('cassandra', 'db_host')], config.get('cassandra', 'keyspace')) model.create(**kwargs)
def parse_counters(data=None): if data is None: data = r.blpop(config.get('constants', 'REDIS_XML_QUEUE_KEY')) LOG.debug(data) data = eval(data[1]) p_obj = Parser.parse_mfc_counters(data[0], data[1], data[2]) r.rpush(config.get('constants', 'REDIS_PARSER_QUEUE_KEY'), Serialize.to_json(p_obj)) return p_obj
def parse_counters(data=None): if data is None: data = r.blpop(config.get('constants', 'REDIS_XML_QUEUE_KEY')) LOG.debug(data) data = eval(data[1]) p_obj = Parser.parse_mfc_counters(data[0], data[1], data[2]) r.rpush(config.get('constants', 'REDIS_PARSER_QUEUE_KEY'), Serialize.to_json(p_obj)) return p_obj
def schedule_events_task(): import schedule from datetime import date, timedelta from analyticsengine.dbmanager.mfc import create_daily_tables unsync_dev_key = config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY') unsync_flag = False """ Job to create daily DB tables calculate the next day's date and pass it to create all the tables for next day. """ def create_daily_cf_job(): tomorrow = date.today()+timedelta(days=1) tomorrow_strf = tomorrow.strftime('%m%d%Y') LOG.info("Creating tables for date: " + tomorrow_strf) create_daily_tables(tomorrow_strf) """ Schedule daily DB table creation. Will create DB tables for next day at 23:30 of every day """ schedule.every().day.at("23:30").do(create_daily_cf_job) """ Job to recheck un-synced devices if unsync_dev_list exist, config request should be sent to see if the device can be moved to sync. devices are popped from unsync list as they are prepared for recheck. when device get send to check config(Sync check), they get added to unsync if its not able to sync. """ def recheck_unsync_devices(): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) recheck_devices = [] while len(unsync_list) > 0: recheck_devices.append(unsync_list.pop()) LOG.info("Processing unsync device list") recheck_task = chain(request_cluster_config.s(recheck_devices), update_unsync_list.s()) recheck_task.apply_async() while True: schedule.run_pending() gevent.sleep(1) if r.exists(unsync_dev_key): if not unsync_flag: LOG.info("Unsync device list found. will schedule job to recheck the status") schedule.every(int(config.get('collector', 'RECHECK_UNSYNC_FREQUENCY'))).minutes.do(recheck_unsync_devices) unsync_flag = True else: LOG.debug("Recheck Unsync devices is already scheduled and is in progress.") else: if unsync_flag: LOG.info("No Unsync devices found. Removing unsync devices rechecking from scheduler") schedule.cancel_job(recheck_unsync_devices) unsync_flag = False
def process_mfc_counters(counters=None, data=None): decimal.getcontext().prec = 6 if counters is None: data = r.blpop(config.get('constants', 'REDIS_PARSER_QUEUE_KEY')) counters = json.loads(data[1]) if counters['data'] is None: LOG.critical("Device: %s, %s IP: %s" % (counters['device_id'], counters['name'], counters['ip'])) LOG.critical( "MFC response doesn't have any counter data. skipping sample: %s" % (counters['sample_id'])) else: gl_bytes = Counter(counters['data']['glbl']['bytes']) # MFC CHR tot_bytes = sum(gl_bytes.values()) tot_cache = counters['data']['glbl']['bytes']['ram'] + counters[ 'data']['glbl']['bytes']['disk'] # Handle Zero condition. Cumulative sum could be 0 if tot_bytes == 0: counters['data']['chr'] = 0 else: counters['data']['chr'] = float( (decimal.Decimal(tot_cache) / decimal.Decimal(tot_bytes)) * 100) #Calculate current throughput mfcs_cur_thrpt = Dict(key=config.get('constants', 'REDIS_MFC_CUR_THRPT_KEY'), redis=r) try: counters['data']['cur_thrpt'] = gl_bytes - mfcs_cur_thrpt[ counters['device_id']] counters['data']['cur_thrpt']['total'] = sum( counters['data']['cur_thrpt'].values()) counters['data']['cur_thrpt']['cache'] = counters['data']['cur_thrpt']['ram'] + \ counters['data']['cur_thrpt']['disk'] mfcs_cur_thrpt[counters['device_id']] = gl_bytes except KeyError: LOG.debug("current throughput hashmap - Initial update for " + str(counters['device_id'])) counters['data']['cur_thrpt'] = mfcs_cur_thrpt[ counters['device_id']] = gl_bytes counters['data']['cur_thrpt']['total'] = counters['data'][ 'cur_thrpt']['cache'] = 0 r.rpush(config.get('constants', 'REDIS_MFC_STORE_QUEUE_KEY'), json.dumps(counters)) return counters
def connect_cassandra(): error = False cluster = Cluster([config.get('cassandra', 'db_host')], port=config.get('cassandra', 'db_port'), protocol_version=3, idle_heartbeat_interval=120) try: LOG.info("Connecting to Cassandra..") return cluster.connect(config.get('cassandra', 'keyspace')) except NoHostAvailable: error = True LOG.info("ERROR: Check Cassandra connection settings in conf") except InvalidRequest: LOG.info("ERROR: Could not find existing Cassandra keyspace. will create new one") try: db_connection = cluster.connect() CREATE_KEYSPACE = """ CREATE KEYSPACE %s WITH replication = {'class': '%s', 'replication_factor': %s } """ % (config.get('cassandra', 'keyspace'), config.get('cassandra', 'replication_strategy'), config.get('cassandra', 'replication_factor')) db_connection.execute(CREATE_KEYSPACE) db_connection.set_keyspace(config.get('cassandra', 'keyspace')) LOG.info("Created and session set to new keyspace: %s" % config.get('cassandra', 'keyspace')) return db_connection except SyntaxException: error = True LOG.info("ERROR: couldn't create new keyspace. check keyspace settings in conf. Exiting now.") raise except: error = True LOG.info("ERROR: something wrong with Cassandra connection") finally: if error: LOG.info("Exiting..") sys.exit(0)
def update_unsync_list(sync_list): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) for device in unsync_list: if device in sync_list: unsync_list.remove(device)
class DailyClusterCounters(Model): __keyspace__ = config.get('cassandra', 'keyspace') __table_name__ = 'cluster_' + today.strftime('%m%d%Y') name = columns.Text(primary_key=True, partition_key=True) ts = columns.DateTime(primary_key=True) value = columns.Map(columns.Text, columns.BigInt) sample_id = columns.Text()
def parse_cluster_stats(): xml_q = config.get('constants', 'REDIS_XML_QUEUE_KEY') while True: data = r.blpop(xml_q) parse_counters.apply_async(args=[data], queue='parse', routing_key='parse.stats')
def parse_config_and_sync(data=None, sync_list=True): if sync_list: sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) else: sync_list = List(key=config.get('constants', 'REDIS_NEW_SYNC_DEV_LIST_KEY'), redis=r) unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) if data is None: data = r.blpop(config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY')) data = eval(data[1]) device = data[0] xml = data[1] p_obj = Parser.parse_mfc_config(device, xml) if p_obj.header.status_code == 0: """ Update the gloabl DS extend the sync_dev_list with the device tuple Store the UUID in a global hashmap. will be retrieved using IP key. """ try: mfc_uuid.update({device[2] + '_uuid': p_obj.data.config.host_id}) mfc_uuid.update( {device[2] + '_hostname': p_obj.data.config.hostname}) #Update to the sync list if its able to retrieve the data attributes sync_list.extend((device, )) except AttributeError: LOG.error("Something wrong with the Config data from MFC: " + device[2]) LOG.error("Restart agentd or make sure the config data is valid.") unsync_list.extend((device, )) finally: r.rpush(config.get('constants', 'REDIS_CONFIG_STORE_QUEUE_KEY'), Serialize.to_json(p_obj)) else: LOG.error("Unable to get config from MFC: " + device[2]) LOG.error("Status Code: %s Message: %s" % (p_obj.header.status_code, p_obj.header.status_msg)) LOG.error("Check MFC state make sure Agentd is working fine.") unsync_list.extend((device, )) return p_obj
def request_config_mfc(ip, data=None): if data is None: data = """<mfc-request><header><type>GET</type></header> <data>running-config mfc-cluster mfc</data></mfc-request>""" mfc_con = MfcHttpConnection(ip) resp = mfc_con.send_request(data) redis.rpush(config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY'), [ip, resp]) return resp
def recheck_unsync_devices(): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) recheck_devices = [] while len(unsync_list) > 0: recheck_devices.append(unsync_list.pop()) LOG.info("Processing unsync device list") recheck_task = chain(request_cluster_config.s(recheck_devices), update_unsync_list.s()) recheck_task.apply_async()
def get_device_list(get_dev_from_file=False): from analyticsengine import dbmanager if get_dev_from_file: with open(config.get('constants', 'CONF_BASE_PATH') + config.get('constants', 'IP_LIST_FILENAME'), 'r') as fp: dev_list = fp.readlines() mfa_dev_list.extend(dev_list) else: LOG.info("Querying MySQL to get the list of devices..") mysql_db = dbmanager.connect_mysql() mysql_cur = mysql_db.cursor() mysql_cur.execute(config.get('constants', 'MFC_DEV_LIST_QUERY')) rows = mysql_cur.fetchmany(500) while len(rows) > 0: mfa_dev_list.extend(rows) for device in rows: LOG.info("Found Device: %s %s %s" % device) rows = mysql_cur.fetchmany(500) mysql_db.close()
class MfcSummary(Model): __keyspace__ = config.get('cassandra', 'keyspace') __table_name__ = "test" mfcid = columns.Text() ip = columns.Text() type = columns.Text() name = columns.Text() ts = columns.DateTime(primary_key=True) value = columns.Map(columns.Text, columns.BigInt)
def request_config_mfc(ip, data=None): if data is None: data = """<mfc-request><header><type>GET</type></header> <data>running-config mfc-cluster mfc</data></mfc-request>""" mfc_con = MfcHttpConnection(ip) resp = mfc_con.send_request(data) redis.rpush(config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY'), [ip, resp]) return resp
def request_stats_mfc(ip, data=None): cleanup_pattern = re.compile(r"\n\s*") if data is None: data = """<mfc-request><header><type>GET</type></header><data>stats mfc-cluster mfc</data></mfc-request>""" mfc_con = MfcHttpConnection(ip) resp = mfc_con.send_request(data) resp = cleanup_pattern.sub("", resp) if ip not in mfc_uuid.keys(): mfc_uuid[ip] = str(uuid.uuid1()) r.rpush(config.get('constants', 'REDIS_XML_QUEUE_KEY'), [mfc_uuid[ip + '_uuid'], ip, resp]) return resp
def get_device_list(get_dev_from_file=False): from analyticsengine import dbmanager if get_dev_from_file: with open( config.get('constants', 'CONF_BASE_PATH') + config.get('constants', 'IP_LIST_FILENAME'), 'r') as fp: dev_list = fp.readlines() mfa_dev_list.extend(dev_list) else: LOG.info("Querying MySQL to get the list of devices..") mysql_db = dbmanager.connect_mysql() mysql_cur = mysql_db.cursor() mysql_cur.execute(config.get('constants', 'MFC_DEV_LIST_QUERY')) rows = mysql_cur.fetchmany(500) while len(rows) > 0: mfa_dev_list.extend(rows) for device in rows: LOG.info("Found Device: %s %s %s" % device) rows = mysql_cur.fetchmany(500) mysql_db.close()
def request_cluster_config(dev_list, unsync_list=False): req_uri = '/admin/agentd_comm' conf_q = config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY') mfc_count = len(dev_list) g_pool = gevent.pool.Pool(size=mfc_count) sync_flag = True if unsync_list: sync_flag = False LOG.debug("Creating Config request clients") conf_clients = [] for device in dev_list: url = URL('http://' + device[2] + ':8080' + req_uri) conf_clients.append( HTTPClient.from_url(url, concurrency=1, headers_type=dict)) LOG.debug("Starting to request Config from MFC") for i in xrange(mfc_count): g_pool.spawn(request_config_mfc_cb, conf_clients[i], dev_list[i], conf_q) g_pool.join() LOG.debug("Finished collecting Config from MFC") for i in xrange(mfc_count): conf_clients[i].close() """Parse and store the config. mfc_uuid is a global hashmap(redis Dict) with ip as key and UUID as value parse_config_and_sync will update the sync_dev_list, mfc_uuid for each XML response. """ LOG.debug("Parsing config request output and building the UUID hash.") q_len = r.llen(conf_q) g_pool = gevent.pool.Pool(size=q_len) for _ in xrange(q_len): data = r.blpop(conf_q) g_pool.spawn(parse_config_and_sync, data, sync_flag) g_pool.join() """Return list of MFCs which was able to communicate.""" sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) return list(sync_list)
def request_stats_mfc(ip, data=None): cleanup_pattern = re.compile(r"\n\s*") if data is None: data = """<mfc-request><header><type>GET</type></header><data>stats mfc-cluster mfc</data></mfc-request>""" mfc_con = MfcHttpConnection(ip) resp = mfc_con.send_request(data) resp = cleanup_pattern.sub("", resp) if ip not in mfc_uuid.keys(): mfc_uuid[ip] = str(uuid.uuid1()) r.rpush(config.get('constants', 'REDIS_XML_QUEUE_KEY'), [mfc_uuid[ip + '_uuid'], ip, resp]) return resp
def recheck_unsync_devices(): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) recheck_devices = [] while len(unsync_list) > 0: recheck_devices.append(unsync_list.pop()) LOG.info("Processing unsync device list") recheck_task = chain(request_cluster_config.s(recheck_devices), update_unsync_list.s()) recheck_task.apply_async()
def request_cluster_config(dev_list, unsync_list=False): req_uri = '/admin/agentd_comm' conf_q = config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY') mfc_count = len(dev_list) g_pool = gevent.pool.Pool(size=mfc_count) sync_flag = True if unsync_list: sync_flag = False LOG.debug("Creating Config request clients") conf_clients = [] for device in dev_list: url = URL('http://' + device[2] + ':8080' + req_uri) conf_clients.append(HTTPClient.from_url(url, concurrency=1, headers_type=dict)) LOG.debug("Starting to request Config from MFC") for i in xrange(mfc_count): g_pool.spawn(request_config_mfc_cb, conf_clients[i], dev_list[i], conf_q) g_pool.join() LOG.debug("Finished collecting Config from MFC") for i in xrange(mfc_count): conf_clients[i].close() """Parse and store the config. mfc_uuid is a global hashmap(redis Dict) with ip as key and UUID as value parse_config_and_sync will update the sync_dev_list, mfc_uuid for each XML response. """ LOG.debug("Parsing config request output and building the UUID hash.") q_len = r.llen(conf_q) g_pool = gevent.pool.Pool(size=q_len) for _ in xrange(q_len): data = r.blpop(conf_q) g_pool.spawn(parse_config_and_sync, data, sync_flag) g_pool.join() """Return list of MFCs which was able to communicate.""" sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) return list(sync_list)
def parse_config_and_sync(data=None, sync_list=True): if sync_list: sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) else: sync_list = List(key=config.get('constants', 'REDIS_NEW_SYNC_DEV_LIST_KEY'), redis=r) unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) if data is None: data = r.blpop(config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY')) data = eval(data[1]) device = data[0] xml = data[1] p_obj = Parser.parse_mfc_config(device, xml) if p_obj.header.status_code == 0: """ Update the gloabl DS extend the sync_dev_list with the device tuple Store the UUID in a global hashmap. will be retrieved using IP key. """ try: mfc_uuid.update({device[2] + '_uuid': p_obj.data.config.host_id}) mfc_uuid.update({device[2] + '_hostname': p_obj.data.config.hostname}) #Update to the sync list if its able to retrieve the data attributes sync_list.extend((device,)) except AttributeError: LOG.error("Something wrong with the Config data from MFC: " + device[2]) LOG.error("Restart agentd or make sure the config data is valid.") unsync_list.extend((device,)) finally: r.rpush(config.get('constants', 'REDIS_CONFIG_STORE_QUEUE_KEY'), Serialize.to_json(p_obj)) else: LOG.error("Unable to get config from MFC: " + device[2]) LOG.error("Status Code: %s Message: %s" % (p_obj.header.status_code, p_obj.header.status_msg)) LOG.error("Check MFC state make sure Agentd is working fine.") unsync_list.extend((device,)) return p_obj
def process_mfc_counters(counters=None, data=None): decimal.getcontext().prec = 6 if counters is None: data = r.blpop(config.get('constants', 'REDIS_PARSER_QUEUE_KEY')) counters = json.loads(data[1]) if counters['data'] is None: LOG.critical("Device: %s, %s IP: %s" % (counters['device_id'], counters['name'], counters['ip'])) LOG.critical("MFC response doesn't have any counter data. skipping sample: %s" % (counters['sample_id'])) else: gl_bytes = Counter(counters['data']['glbl']['bytes']) # MFC CHR tot_bytes = sum(gl_bytes.values()) tot_cache = counters['data']['glbl']['bytes']['ram'] + counters['data']['glbl']['bytes']['disk'] # Handle Zero condition. Cumulative sum could be 0 if tot_bytes == 0: counters['data']['chr'] = 0 else: counters['data']['chr'] = float((decimal.Decimal(tot_cache) / decimal.Decimal(tot_bytes)) * 100) #Calculate current throughput mfcs_cur_thrpt = Dict(key=config.get('constants', 'REDIS_MFC_CUR_THRPT_KEY'), redis=r) try: counters['data']['cur_thrpt'] = gl_bytes - mfcs_cur_thrpt[counters['device_id']] counters['data']['cur_thrpt']['total'] = sum(counters['data']['cur_thrpt'].values()) counters['data']['cur_thrpt']['cache'] = counters['data']['cur_thrpt']['ram'] + \ counters['data']['cur_thrpt']['disk'] mfcs_cur_thrpt[counters['device_id']] = gl_bytes except KeyError: LOG.debug("current throughput hashmap - Initial update for " + str(counters['device_id'])) counters['data']['cur_thrpt'] = mfcs_cur_thrpt[counters['device_id']] = gl_bytes counters['data']['cur_thrpt']['total'] = counters['data']['cur_thrpt']['cache'] = 0 r.rpush(config.get('constants', 'REDIS_MFC_STORE_QUEUE_KEY'), json.dumps(counters)) return counters
def store_mfc_config(): from analyticsengine.dbmanager.mfc.schema import MFC_CONFIG_TABLE_NAME from analyticsengine import dbmanager from datetime import datetime db_connection = dbmanager.connect_cassandra() CONFIG_TABLE_INSERT = "INSERT INTO " + MFC_CONFIG_TABLE_NAME + """ (mfcid, hostname, ip, ts, type, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(ts)s, %(type)s, %(value)s) """ timestamp = lambda dt: long( (dt - datetime.fromtimestamp(0)).total_seconds() * 1000) while True: data = r.blpop(config.get('constants', 'REDIS_CONFIG_STORE_QUEUE_KEY')) conf_data = json.loads(data[1]) pk = conf = raw = dict() #Primary Key. pk['mfcid'] = str(conf_data['device_id']) pk['hostname'] = conf_data['name'] pk['ip'] = conf_data['ip'] pk['ts'] = timestamp(datetime.now()) conf.update(pk) conf['type'] = 'config' conf['value'] = dict({ 'host_id': conf_data['data']['config']['host_id'], 'version': conf_data['data']['config']['version'], 'licenses': json.dumps(conf_data['data']['config']['licenses']), 'network': json.dumps(conf_data['data']['config']['network']), 'ifcfg': json.dumps(conf_data['data']['config']['ifcfg']), 'namespaces': json.dumps(conf_data['data']['config']['namespaces']), }) db_connection.execute(CONFIG_TABLE_INSERT, conf) raw.update(pk) raw['type'] = 'raw' raw['value'] = dict( {'dump': json.dumps(conf_data['data']['config']['dump'])}) db_connection.execute(CONFIG_TABLE_INSERT, raw)
def connect_cassandra(): error = False cluster = Cluster([config.get('cassandra', 'db_host')], port=config.get('cassandra', 'db_port'), protocol_version=3, idle_heartbeat_interval=120) try: LOG.info("Connecting to Cassandra..") return cluster.connect(config.get('cassandra', 'keyspace')) except NoHostAvailable: error = True LOG.info("ERROR: Check Cassandra connection settings in conf") except InvalidRequest: LOG.info( "ERROR: Could not find existing Cassandra keyspace. will create new one" ) try: db_connection = cluster.connect() CREATE_KEYSPACE = """ CREATE KEYSPACE %s WITH replication = {'class': '%s', 'replication_factor': %s } """ % ( config.get('cassandra', 'keyspace'), config.get('cassandra', 'replication_strategy'), config.get('cassandra', 'replication_factor')) db_connection.execute(CREATE_KEYSPACE) db_connection.set_keyspace(config.get('cassandra', 'keyspace')) LOG.info("Created and session set to new keyspace: %s" % config.get('cassandra', 'keyspace')) return db_connection except SyntaxException: error = True LOG.info( "ERROR: couldn't create new keyspace. check keyspace settings in conf. Exiting now." ) raise except: error = True LOG.info("ERROR: something wrong with Cassandra connection") finally: if error: LOG.info("Exiting..") sys.exit(0)
def run_request_fetch(): LOG.info("Starting request fetch task") req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) """Will split fetch job in to chain with two tasks. First request the config for all the MFCs. Prepare the list of MFCs that can be accessed. A hashmap of IP with UUID is created pass the SYNCd MFCs to fetch the stats. """ if len(mfa_dev_list) > 0: collector_task = chain( request_cluster_config.s(list(mfa_dev_list)), request_cluster_stats.s(interval=req_interval), ) collector_task.apply_async() else: LOG.error("Devices list not found. Check file or MFA DB") sys.exit(0) '''
def connect_mysql(): import MySQLdb import _mysql_exceptions try: mysql_db = MySQLdb.connect(host=config.get('mysql', 'db_host'), port=int(config.get('mysql', 'db_port')), user=config.get('mysql', 'db_user'), passwd=config.get('mysql', 'db_password'), db=config.get('mysql', 'db_name'), connect_timeout=config.get('constants', 'MYSQL_CONNECT_TIMEOUT'), charset='utf8', use_unicode=True) return mysql_db except _mysql_exceptions.OperationalError as e: LOG.error(e)
def run_request_fetch(): LOG.info("Starting request fetch task") req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) """Will split fetch job in to chain with two tasks. First request the config for all the MFCs. Prepare the list of MFCs that can be accessed. A hashmap of IP with UUID is created pass the SYNCd MFCs to fetch the stats. """ if len(mfa_dev_list) > 0: collector_task = chain(request_cluster_config.s(list(mfa_dev_list)), request_cluster_stats.s(interval=req_interval), ) collector_task.apply_async() else: LOG.error("Devices list not found. Check file or MFA DB") sys.exit(0) '''
def store_mfc_config(): from analyticsengine.dbmanager.mfc.schema import MFC_CONFIG_TABLE_NAME from analyticsengine import dbmanager from datetime import datetime db_connection = dbmanager.connect_cassandra() CONFIG_TABLE_INSERT = "INSERT INTO " + MFC_CONFIG_TABLE_NAME + """ (mfcid, hostname, ip, ts, type, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(ts)s, %(type)s, %(value)s) """ timestamp = lambda dt: long((dt - datetime.fromtimestamp(0)).total_seconds() * 1000) while True: data = r.blpop(config.get('constants', 'REDIS_CONFIG_STORE_QUEUE_KEY')) conf_data = json.loads(data[1]) pk = conf = raw = dict() #Primary Key. pk['mfcid'] = str(conf_data['device_id']) pk['hostname'] = conf_data['name'] pk['ip'] = conf_data['ip'] pk['ts'] = timestamp(datetime.now()) conf.update(pk) conf['type'] = 'config' conf['value'] = dict({ 'host_id': conf_data['data']['config']['host_id'], 'version': conf_data['data']['config']['version'], 'licenses': json.dumps(conf_data['data']['config']['licenses']), 'network': json.dumps(conf_data['data']['config']['network']), 'ifcfg': json.dumps(conf_data['data']['config']['ifcfg']), 'namespaces': json.dumps(conf_data['data']['config']['namespaces']), }) db_connection.execute(CONFIG_TABLE_INSERT, conf) raw.update(pk) raw['type'] = 'raw' raw['value'] = dict({ 'dump': json.dumps(conf_data['data']['config']['dump']) }) db_connection.execute(CONFIG_TABLE_INSERT, raw)
def connect_mysql(): import MySQLdb import _mysql_exceptions try: mysql_db = MySQLdb.connect(host=config.get('mysql', 'db_host'), port=int(config.get('mysql', 'db_port')), user=config.get('mysql', 'db_user'), passwd=config.get('mysql', 'db_password'), db=config.get('mysql', 'db_name'), connect_timeout=config.get( 'constants', 'MYSQL_CONNECT_TIMEOUT'), charset='utf8', use_unicode=True) return mysql_db except _mysql_exceptions.OperationalError as e: LOG.error(e)
def request_cluster_stats(sync_mfcs, interval=20): req_uri = '/admin/agentd_comm' xml_q = config.get('constants', 'REDIS_XML_QUEUE_KEY') new_dev_list_key = config.get('constants', 'REDIS_NEW_FOUND_DEV_LIST_KEY') new_sync_dev_list_key = config.get('constants', 'REDIS_NEW_SYNC_DEV_LIST_KEY') sync_mfcs_key = config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY') signal.signal(signal.SIGQUIT, gevent.kill) stat_clients = [] """Request to synced MFCs will get the IP list from mfc_uuid """ sync_mfcs_count = len(sync_mfcs) LOG.info("Synced MFCs: ") for device_id, name, ip in sync_mfcs: LOG.info("%s %s %s" % (device_id, name, ip)) def create_stat_clients(): LOG.info("Creating Stats request clients") for device_id, name, ip in sync_mfcs: url = URL('http://' + ip + ':8080' + req_uri) stat_clients.append(HTTPClient.from_url(url, concurrency=1, headers_type=dict)) def close_stat_clients(): for c in xrange(sync_mfcs_count): stat_clients[c].close() create_stat_clients() g_req_pool = gevent.pool.Pool(size=sync_mfcs_count) LOG.info("Starting to request stats from MFC") while True: # Commented following : time based check hogs CPU cycles. ''' if tick(t1) >= interval or initial: t1 = time.time() initial = False ''' sample_id = str(uuid.uuid1()) for i in xrange(sync_mfcs_count): g_req_pool.spawn(request_stats_mfc_cb, stat_clients[i], sync_mfcs[i], sample_id, xml_q) g_req_pool.join(timeout=interval) gevent.sleep(interval) if r.exists(new_sync_dev_list_key): LOG.info("New MFCs added to the Sync list- updating stat request clients") close_stat_clients() stat_clients = [] LOG.info("Newly Synced MFCs: ") new_sync_mfcs = list(List(key=new_sync_dev_list_key, redis=r)) for device_id, name, ip in new_sync_mfcs: LOG.info("%s %s %s" % (device_id, name, ip)) r.delete(new_dev_list_key) #Get the current synced list and extend with newly synced list sync_mfcs = List(key=sync_mfcs_key, redis=r) sync_mfcs.extend(new_sync_mfcs) sync_mfcs = list(sync_mfcs) sync_mfcs_count = len(sync_mfcs) create_stat_clients() close_stat_clients()
raise #AELOG rotating file handler self.aelog_fh = handlers.RotatingFileHandler( self.aelog_path + self.aelog_file_name, maxBytes=self.aelog_file_size, backupCount=self.aelog_file_count) self.aelog_fh.setLevel(self.aelog_fh_level) # Logging format def set_format(self, log_format): if log_format is None: log_format = '%(asctime)s - %(funcName)s - %(levelname)s - %(message)s' self.aelog_format = logging.Formatter(log_format) self.aelog_fh.setFormatter(self.aelog_format) def set_fh(self, fh=None): if fh is not None: self.aelog_fh = fh if self.aelog_fh is not None: self.addHandler(self.aelog_fh) logging.setLoggerClass(AELog) LOG = logging.getLogger('MF-AE-LOG') LOG.setLevel(eval('logging.' + config.get('logger', 'level'))) LOG.create_fh(config.get('logger', 'file_path'), config.get('logger', 'file_name'), eval(config.get('logger', 'fh_log_level'))) LOG.set_format(config.get('logger', 'format')) LOG.set_fh()
def schedule_events_task(): import schedule from datetime import date, timedelta from analyticsengine.dbmanager.mfc import create_daily_tables unsync_dev_key = config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY') unsync_flag = False """ Job to create daily DB tables calculate the next day's date and pass it to create all the tables for next day. """ def create_daily_cf_job(): tomorrow = date.today() + timedelta(days=1) tomorrow_strf = tomorrow.strftime('%m%d%Y') LOG.info("Creating tables for date: " + tomorrow_strf) create_daily_tables(tomorrow_strf) """ Schedule daily DB table creation. Will create DB tables for next day at 23:30 of every day """ schedule.every().day.at("23:30").do(create_daily_cf_job) """ Job to recheck un-synced devices if unsync_dev_list exist, config request should be sent to see if the device can be moved to sync. devices are popped from unsync list as they are prepared for recheck. when device get send to check config(Sync check), they get added to unsync if its not able to sync. """ def recheck_unsync_devices(): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) recheck_devices = [] while len(unsync_list) > 0: recheck_devices.append(unsync_list.pop()) LOG.info("Processing unsync device list") recheck_task = chain(request_cluster_config.s(recheck_devices), update_unsync_list.s()) recheck_task.apply_async() while True: schedule.run_pending() gevent.sleep(1) if r.exists(unsync_dev_key): if not unsync_flag: LOG.info( "Unsync device list found. will schedule job to recheck the status" ) schedule.every( int(config.get('collector', 'RECHECK_UNSYNC_FREQUENCY'))).minutes.do( recheck_unsync_devices) unsync_flag = True else: LOG.debug( "Recheck Unsync devices is already scheduled and is in progress." ) else: if unsync_flag: LOG.info( "No Unsync devices found. Removing unsync devices rechecking from scheduler" ) schedule.cancel_job(recheck_unsync_devices) unsync_flag = False
from celery import chain import decimal from analyticsengine.celeryapp.celery import celery from analyticsengine.collector.http import MfcHttpConnection from analyticsengine.config import config from analyticsengine.logging import LOG from analyticsengine.parser import Parser from analyticsengine.parser.mfc.common import Serialize from geventhttpclient import httplib httplib.patch() from geventhttpclient import HTTPClient, URL #redis.connection.socket = gevent.socket r = redis.Redis(host=config.get('redis', 'db_host'), port=config.get('redis', 'db_port'), db=config.get('redis', 'db')) r_keys = { 'dev_list': config.get('constants', 'REDIS_DEV_LIST_KEY'), 'sync_dev_list': config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), 'mfc_uuid': config.get('constants', 'REDIS_MFC_UUID_HASH_KEY'), 'cur_thrpt': config.get('constants', 'REDIS_MFC_CUR_THRPT_KEY'), 'unsync_dev_list': config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), 'new_found_dev_list': config.get('constants', 'REDIS_NEW_FOUND_DEV_LIST_KEY') } mfa_dev_list = List(key=r_keys['dev_list'], redis=r) sync_dev_list = List(key=r_keys['sync_dev_list'], redis=r) mfc_uuid = Dict(key=r_keys['mfc_uuid'], redis=r) """Cluster Request Tasks"""
def process_cluster_counters(counters): r.rpush(config.get('constants', 'REDIS_CLUSTER_BUCKET_QUEUE_KEY'), counters)
def request_cluster_stats(sync_mfcs, interval=20): req_uri = '/admin/agentd_comm' xml_q = config.get('constants', 'REDIS_XML_QUEUE_KEY') new_dev_list_key = config.get('constants', 'REDIS_NEW_FOUND_DEV_LIST_KEY') new_sync_dev_list_key = config.get('constants', 'REDIS_NEW_SYNC_DEV_LIST_KEY') sync_mfcs_key = config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY') signal.signal(signal.SIGQUIT, gevent.kill) stat_clients = [] """Request to synced MFCs will get the IP list from mfc_uuid """ sync_mfcs_count = len(sync_mfcs) LOG.info("Synced MFCs: ") for device_id, name, ip in sync_mfcs: LOG.info("%s %s %s" % (device_id, name, ip)) def create_stat_clients(): LOG.info("Creating Stats request clients") for device_id, name, ip in sync_mfcs: url = URL('http://' + ip + ':8080' + req_uri) stat_clients.append( HTTPClient.from_url(url, concurrency=1, headers_type=dict)) def close_stat_clients(): for c in xrange(sync_mfcs_count): stat_clients[c].close() create_stat_clients() g_req_pool = gevent.pool.Pool(size=sync_mfcs_count) LOG.info("Starting to request stats from MFC") while True: # Commented following : time based check hogs CPU cycles. ''' if tick(t1) >= interval or initial: t1 = time.time() initial = False ''' sample_id = str(uuid.uuid1()) for i in xrange(sync_mfcs_count): g_req_pool.spawn(request_stats_mfc_cb, stat_clients[i], sync_mfcs[i], sample_id, xml_q) g_req_pool.join(timeout=interval) gevent.sleep(interval) if r.exists(new_sync_dev_list_key): LOG.info( "New MFCs added to the Sync list- updating stat request clients" ) close_stat_clients() stat_clients = [] LOG.info("Newly Synced MFCs: ") new_sync_mfcs = list(List(key=new_sync_dev_list_key, redis=r)) for device_id, name, ip in new_sync_mfcs: LOG.info("%s %s %s" % (device_id, name, ip)) r.delete(new_dev_list_key) #Get the current synced list and extend with newly synced list sync_mfcs = List(key=sync_mfcs_key, redis=r) sync_mfcs.extend(new_sync_mfcs) sync_mfcs = list(sync_mfcs) sync_mfcs_count = len(sync_mfcs) create_stat_clients() close_stat_clients()
def update_unsync_list(sync_list): unsync_list = List(key=config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), redis=r) for device in unsync_list: if device in sync_list: unsync_list.remove(device)
from celery import chain import decimal from analyticsengine.celeryapp.celery import celery from analyticsengine.collector.http import MfcHttpConnection from analyticsengine.config import config from analyticsengine.logging import LOG from analyticsengine.parser import Parser from analyticsengine.parser.mfc.common import Serialize from geventhttpclient import httplib httplib.patch() from geventhttpclient import HTTPClient, URL #redis.connection.socket = gevent.socket r = redis.Redis(host=config.get('redis', 'db_host'), port=config.get('redis', 'db_port'), db=config.get('redis', 'db')) r_keys = { 'dev_list': config.get('constants', 'REDIS_DEV_LIST_KEY'), 'sync_dev_list': config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), 'mfc_uuid': config.get('constants', 'REDIS_MFC_UUID_HASH_KEY'), 'cur_thrpt': config.get('constants', 'REDIS_MFC_CUR_THRPT_KEY'), 'unsync_dev_list': config.get('constants', 'REDIS_UNSYNC_DEV_LIST_KEY'), 'new_found_dev_list': config.get('constants', 'REDIS_NEW_FOUND_DEV_LIST_KEY') } mfa_dev_list = List(key=r_keys['dev_list'], redis=r) sync_dev_list = List(key=r_keys['sync_dev_list'], redis=r) mfc_uuid = Dict(key=r_keys['mfc_uuid'], redis=r) """Cluster Request Tasks"""
def process_cluster_stats(): def multi_dict_counter(level): if level < 1: return Counter() return defaultdict(lambda: multi_dict_counter(level - 1)) """ Creating a 2D dictionary to hold the cluster wide counter counters from across MFCs will be aggregated based on the sample ID. cluster[<Sample ID>][<Counter Name>] = Counter(<Dict of counter values>) cluster['cumulative'][<Counter Name>] will be used to keep track of the cumulative of last sample Delta will be calculated using above counter. """ cluster = multi_dict_counter(2) # 2 Level dictionary of Counter tick = lambda x: time.time() - x item_cnt = 0 cur_sample = None #mfc_hash = Dict(key=config.get('constants', 'REDIS_MFC_UUID_HASH_KEY'), redis=r) sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) cluster_sample_timeout = config.get('constants', 'CLUSTER_SAMPLE_TIMEOUT') store_q = config.get('constants', 'REDIS_CLUSTER_STORE_QUEUE_KEY') req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) sample_q = [] while True: data = r.blpop(config.get('constants', 'REDIS_PARSER_QUEUE_KEY')) counters = json.loads(data[1]) #Check if data exist for the parsed response. Agentd response can be empty if counters['data'] is not None: """Process each MFC counter.""" process_mfc_counters.apply_async(args=[counters], queue='process', routing_key='process.stat') """Process Cluster wide cumulative data for same sample ID.""" item_cnt += 1 # Requests cluster[counters['sample_id']]['requests'].update( counters['data']['glbl']['requests']) #Cumulative Bytes cluster[counters['sample_id']]['bytes'].update( counters['data']['glbl']['bytes']) #Timestamp cluster[counters['sample_id']]['timestamp'] = counters['data'][ 'timestamp'] try: cluster[counters['sample_id']]['ip_list'].append( counters['ip']) # Preserve the IP except AttributeError: cluster[counters['sample_id']]['ip_list'] = list() cluster[counters['sample_id']]['ip_list'].append( counters['ip']) if cur_sample is not None and cur_sample != counters['sample_id']: # new sample has arrived if item_cnt > len(sync_list) or tick( init_sample_ts) >= cluster_sample_timeout: # 1st case: record from all the Sync'd MFCs received. Store and remove the sample from cluster DS. # or 2nd case: some data still left to be received but hit sample time out. #Calculate cumulative Delta. cluster[cur_sample]['cur_thrpt'] = cluster[cur_sample][ 'bytes'] - cluster['cumulative']['bytes'] cluster[cur_sample]['cur_thrpt']['total'] = sum( cluster[cur_sample]['cur_thrpt'].values()) cluster[cur_sample]['cur_thrpt']['cache'] = cluster[cur_sample]['cur_thrpt']['ram'] + \ cluster[cur_sample]['cur_thrpt']['disk'] #Preserve the cumulative for next sample set cluster['cumulative']['bytes'] = cluster[cur_sample][ 'bytes'] #Push to store the data r.rpush(store_q, (cur_sample, dict(cluster[cur_sample]))) del cluster[cur_sample] item_cnt = 1 cur_sample = sample_q.pop(0) if ( len(sample_q) > 0) else counters['sample_id'] init_sample_ts = time.time() else: LOG.info( "Got new sample ID: %s. Need to wait for current sample(%s) to arrive until pushed out" % (counters['sample_id'], cur_sample)) LOG.info("Adding sample ID to the waiting list.") if counters['sample_id'] not in sample_q: sample_q.append(counters['sample_id']) if cur_sample is None: cur_sample = counters['sample_id'] init_sample_ts = time.time() else: LOG.critical( "Device: %s, %s IP: %s" % (counters['device_id'], counters['name'], counters['ip'])) LOG.critical( "MFC response doesn't have any counter data. skipping sample: %s" % (counters['sample_id']))
def process_cluster_counters(counters): r.rpush(config.get('constants', 'REDIS_CLUSTER_BUCKET_QUEUE_KEY'), counters)
def store_mfc_stats(): from datetime import date from analyticsengine import dbmanager from analyticsengine.dbmanager.mfc.schema import MFC_STATS_TABLE_NAME, MFC_SUMMARY_TABLE_NAME db_connection = dbmanager.connect_cassandra() date_strf = lambda dt: dt.strftime('%m%d%Y') req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) while True: data = r.blpop(config.get('constants', 'REDIS_MFC_STORE_QUEUE_KEY')) counters = json.loads(data[1]) """ CF date suffix CF date suffix are calculated based on the timestamp in the response. data can be buffered in queue and may be of different date. Instead of calculating based on current date, its best to stick to the timestamp in the payload as it will then get stored in right CF. This will address the case when queue has buffered data of different date and can recover on a app crash. """ date_str = date_strf(date.fromtimestamp(counters['data']['timestamp'])) DAILY_TABLE_INSERT = "INSERT INTO " + MFC_STATS_TABLE_NAME + date_str + \ """ (mfcid, hostname, ip, ts, type, name, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(ts)s, %(type)s, %(name)s, %(value)s) """ DAILY_SUMMARY_INSERT = "INSERT INTO " + MFC_SUMMARY_TABLE_NAME + date_str + \ """ (mfcid, hostname, ip, sample_id, ts, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(sample_id)s, %(ts)s, %(value)s) """ pk = glbl_bytes = glbl_ds = glbl_ram = glbl_req = glbl_tier = http_ns = sys_stat = dict() """Global stats.""" #Primary Key. pk['mfcid'] = str(counters['device_id']) pk['hostname'] = counters['name'] pk['ip'] = counters['ip'] pk['ts'] = counters['data']['timestamp'] * 1000 #Global Bytes. glbl_bytes.update(pk) glbl_bytes['type'] = 'global' glbl_bytes['name'] = 'bytes' glbl_bytes['value'] = counters['data']['glbl']['bytes'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_bytes, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_bytes) db_connection.execute(DAILY_TABLE_INSERT, glbl_bytes) #Global Disk Space. glbl_ds.update(pk) glbl_ds['type'] = 'global' glbl_ds['name'] = 'disk_space' #Disk space is in MB. Converting to Bytes for k, v in counters['data']['glbl']['disk_space'].items(): counters['data']['glbl']['disk_space'][k] = v * 1024 * 1024 glbl_ds['value'] = counters['data']['glbl']['disk_space'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_ds, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_ds) db_connection.execute(DAILY_TABLE_INSERT, glbl_ds) #Global Ram Cache. glbl_ram.update(pk) glbl_ram['type'] = 'global' glbl_ram['name'] = 'ram_cache' glbl_ram['value'] = counters['data']['glbl']['ram_cache'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_ram, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_ram) db_connection.execute(DAILY_TABLE_INSERT, glbl_ram) #Global Requests. glbl_req.update(pk) glbl_req['type'] = 'global' glbl_req['name'] = 'requests' glbl_req['value'] = counters['data']['glbl']['requests'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_req, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_req) db_connection.execute(DAILY_TABLE_INSERT, glbl_req) #Global Tiers. glbl_tier.update(pk) glbl_tier['type'] = 'global' for tier in counters['data']['glbl']['tiers']: glbl_tier['name'] = tier['provider'] tier.pop('provider') glbl_tier['value'] = tier db_connection.execute(DAILY_TABLE_INSERT, glbl_tier) """Namespace Stats.""" http_ns.update(pk) http_ns['type'] = 'http_ns' for ns in counters['data']['services']['http']['namespaces']: http_ns['name'] = ns['name'] + ':requests' http_ns['value'] = ns['requests'] db_connection.execute(DAILY_TABLE_INSERT, http_ns) http_ns['name'] = ns['name'] + ':bytes' http_ns['value'] = ns['bytes'] db_connection.execute(DAILY_TABLE_INSERT, http_ns) """System Stats.""" sys_stat.update(pk) sys_stat['type'] = 'system' sys_stat['name'] = 'cpu' sys_stat['value'] = counters['data']['system']['cpu'] db_connection.execute(DAILY_TABLE_INSERT, sys_stat) sys_stat['name'] = 'memory' #System Memory is in KB. Converting to Bytes for k, v in counters['data']['system']['memory'].items(): counters['data']['system']['memory'][k] = v * 1024 sys_stat['value'] = counters['data']['system']['memory'] db_connection.execute(DAILY_TABLE_INSERT, sys_stat) """MFC Summary Stats""" sum_stats = dict() sum_stats['mfcid'] = str(counters['device_id']) sum_stats['hostname'] = counters['name'] sum_stats['ip'] = counters['ip'] sum_stats['sample_id'] = counters['sample_id'] sum_stats['ts'] = counters['data']['timestamp'] * 1000 sum_stats['value'] = {} for k, v in counters['data']['glbl']['requests'].items(): sum_stats['value'].update({'req_' + k: str(v)}) for k, v in counters['data']['cur_thrpt'].items(): sum_stats['value'].update({'cur_' + k: str(v/req_interval)}) sum_stats['value'].update({'chr': str(counters['data']['chr'])}) db_connection.execute(DAILY_SUMMARY_INSERT, sum_stats)
def store_cluster_stats(): from datetime import date from analyticsengine.dbmanager.mfc.schema import ( CLUSTER_STATS_TABLE_NAME, CLUSTER_SUMMARY_TABLE_NAME, CLUSTER_SAMPLE_MAP_TABLE_NAME) from analyticsengine import dbmanager from collections import Counter db_connection = dbmanager.connect_cassandra() date_strf = lambda dt: dt.strftime('%m%d%Y') while True: data = r.blpop(config.get('constants', 'REDIS_CLUSTER_STORE_QUEUE_KEY')) sample_id, counters = eval(data[1]) #CF date suffix calculation based on the current timestamp in the payload. Read above for more info. date_str = date_strf(date.fromtimestamp(counters['timestamp'])) DAILY_TABLE_INSERT = "INSERT INTO " + CLUSTER_STATS_TABLE_NAME + date_str + \ """ (name, ts, sample_id, value) VALUES (%(name)s, %(ts)s, %(sample_id)s, %(value)s) """ DAILY_SUMMARY_TABLE_INSERT = "INSERT INTO " + CLUSTER_SUMMARY_TABLE_NAME + date_str + \ """ (name, ts, sample_id, value) VALUES (%(name)s, %(ts)s, %(sample_id)s, %(value)s) """ SAMPLE_MAP_INSERT = "INSERT INTO " + CLUSTER_SAMPLE_MAP_TABLE_NAME + date_str + \ """ (sample_id, ts, ip_list) VALUES (%(sample_id)s, %(ts)s, %(ip_list)s) """ pk = glbl_req = glbl_bytes = sample_map = dict() pk['ts'] = counters['timestamp'] * 1000 glbl_req.update(pk) glbl_req['name'] = 'gl_requests' glbl_req['value'] = dict(counters['requests']) glbl_req['sample_id'] = sample_id #DailyClusterCounters.create(**glbl_req) db_connection.execute(DAILY_TABLE_INSERT, glbl_req) glbl_bytes.update(pk) glbl_bytes['name'] = 'gl_bytes' glbl_bytes['value'] = dict(counters['bytes']) glbl_bytes['sample_id'] = sample_id #DailyClusterCounters.create(**glbl_bytes) db_connection.execute(DAILY_TABLE_INSERT, glbl_bytes) sample_map.update(pk) sample_map['sample_id'] = sample_id sample_map['ip_list'] = counters['ip_list'] db_connection.execute(SAMPLE_MAP_INSERT, sample_map) #Update clusterwide summary cluster_sum = dict() cluster_sum['name'] = 'cur_thrpt' cluster_sum['ts'] = pk['ts'] cluster_sum['sample_id'] = sample_id cluster_sum['value'] = dict(counters['cur_thrpt']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum) cluster_sum['name'] = 'cum_bytes' cluster_sum['value'] = dict(counters['bytes']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum) cluster_sum['name'] = 'requests' cluster_sum['value'] = dict(counters['requests']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum)
os.makedirs(file_path) except OSError as e: if e.errno != errno.EEXIST: raise #AELOG rotating file handler self.aelog_fh = handlers.RotatingFileHandler(self.aelog_path + self.aelog_file_name, maxBytes=self.aelog_file_size, backupCount=self.aelog_file_count) self.aelog_fh.setLevel(self.aelog_fh_level) # Logging format def set_format(self, log_format): if log_format is None: log_format = '%(asctime)s - %(funcName)s - %(levelname)s - %(message)s' self.aelog_format = logging.Formatter(log_format) self.aelog_fh.setFormatter(self.aelog_format) def set_fh(self, fh=None): if fh is not None: self.aelog_fh = fh if self.aelog_fh is not None: self.addHandler(self.aelog_fh) logging.setLoggerClass(AELog) LOG = logging.getLogger('MF-AE-LOG') LOG.setLevel(eval('logging.' + config.get('logger', 'level'))) LOG.create_fh(config.get('logger', 'file_path'), config.get('logger', 'file_name'), eval(config.get('logger', 'fh_log_level'))) LOG.set_format(config.get('logger', 'format')) LOG.set_fh()
def store_cluster_stats(): from datetime import date from analyticsengine.dbmanager.mfc.schema import (CLUSTER_STATS_TABLE_NAME, CLUSTER_SUMMARY_TABLE_NAME, CLUSTER_SAMPLE_MAP_TABLE_NAME) from analyticsengine import dbmanager from collections import Counter db_connection = dbmanager.connect_cassandra() date_strf = lambda dt: dt.strftime('%m%d%Y') while True: data = r.blpop(config.get('constants', 'REDIS_CLUSTER_STORE_QUEUE_KEY')) sample_id, counters = eval(data[1]) #CF date suffix calculation based on the current timestamp in the payload. Read above for more info. date_str = date_strf(date.fromtimestamp(counters['timestamp'])) DAILY_TABLE_INSERT = "INSERT INTO " + CLUSTER_STATS_TABLE_NAME + date_str + \ """ (name, ts, sample_id, value) VALUES (%(name)s, %(ts)s, %(sample_id)s, %(value)s) """ DAILY_SUMMARY_TABLE_INSERT = "INSERT INTO " + CLUSTER_SUMMARY_TABLE_NAME + date_str + \ """ (name, ts, sample_id, value) VALUES (%(name)s, %(ts)s, %(sample_id)s, %(value)s) """ SAMPLE_MAP_INSERT = "INSERT INTO " + CLUSTER_SAMPLE_MAP_TABLE_NAME + date_str + \ """ (sample_id, ts, ip_list) VALUES (%(sample_id)s, %(ts)s, %(ip_list)s) """ pk = glbl_req = glbl_bytes = sample_map = dict() pk['ts'] = counters['timestamp'] * 1000 glbl_req.update(pk) glbl_req['name'] = 'gl_requests' glbl_req['value'] = dict(counters['requests']) glbl_req['sample_id'] = sample_id #DailyClusterCounters.create(**glbl_req) db_connection.execute(DAILY_TABLE_INSERT, glbl_req) glbl_bytes.update(pk) glbl_bytes['name'] = 'gl_bytes' glbl_bytes['value'] = dict(counters['bytes']) glbl_bytes['sample_id'] = sample_id #DailyClusterCounters.create(**glbl_bytes) db_connection.execute(DAILY_TABLE_INSERT, glbl_bytes) sample_map.update(pk) sample_map['sample_id'] = sample_id sample_map['ip_list'] = counters['ip_list'] db_connection.execute(SAMPLE_MAP_INSERT, sample_map) #Update clusterwide summary cluster_sum = dict() cluster_sum['name'] = 'cur_thrpt' cluster_sum['ts'] = pk['ts'] cluster_sum['sample_id'] = sample_id cluster_sum['value'] = dict(counters['cur_thrpt']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum) cluster_sum['name'] = 'cum_bytes' cluster_sum['value'] = dict(counters['bytes']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum) cluster_sum['name'] = 'requests' cluster_sum['value'] = dict(counters['requests']) db_connection.execute(DAILY_SUMMARY_TABLE_INSERT, cluster_sum)
def parse_cluster_stats(): xml_q = config.get('constants', 'REDIS_XML_QUEUE_KEY') while True: data = r.blpop(xml_q) parse_counters.apply_async(args=[data], queue='parse', routing_key='parse.stats')
def process_cluster_stats(): def multi_dict_counter(level): if level < 1: return Counter() return defaultdict(lambda: multi_dict_counter(level-1)) """ Creating a 2D dictionary to hold the cluster wide counter counters from across MFCs will be aggregated based on the sample ID. cluster[<Sample ID>][<Counter Name>] = Counter(<Dict of counter values>) cluster['cumulative'][<Counter Name>] will be used to keep track of the cumulative of last sample Delta will be calculated using above counter. """ cluster = multi_dict_counter(2) # 2 Level dictionary of Counter tick = lambda x: time.time() - x item_cnt = 0 cur_sample = None #mfc_hash = Dict(key=config.get('constants', 'REDIS_MFC_UUID_HASH_KEY'), redis=r) sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) cluster_sample_timeout = config.get('constants', 'CLUSTER_SAMPLE_TIMEOUT') store_q = config.get('constants', 'REDIS_CLUSTER_STORE_QUEUE_KEY') req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) sample_q = [] while True: data = r.blpop(config.get('constants', 'REDIS_PARSER_QUEUE_KEY')) counters = json.loads(data[1]) #Check if data exist for the parsed response. Agentd response can be empty if counters['data'] is not None: """Process each MFC counter.""" process_mfc_counters.apply_async(args=[counters], queue='process', routing_key='process.stat') """Process Cluster wide cumulative data for same sample ID.""" item_cnt += 1 # Requests cluster[counters['sample_id']]['requests'].update(counters['data']['glbl']['requests']) #Cumulative Bytes cluster[counters['sample_id']]['bytes'].update(counters['data']['glbl']['bytes']) #Timestamp cluster[counters['sample_id']]['timestamp'] = counters['data']['timestamp'] try: cluster[counters['sample_id']]['ip_list'].append(counters['ip']) # Preserve the IP except AttributeError: cluster[counters['sample_id']]['ip_list'] = list() cluster[counters['sample_id']]['ip_list'].append(counters['ip']) if cur_sample is not None and cur_sample != counters['sample_id']: # new sample has arrived if item_cnt > len(sync_list) or tick(init_sample_ts) >= cluster_sample_timeout: # 1st case: record from all the Sync'd MFCs received. Store and remove the sample from cluster DS. # or 2nd case: some data still left to be received but hit sample time out. #Calculate cumulative Delta. cluster[cur_sample]['cur_thrpt'] = cluster[cur_sample]['bytes'] - cluster['cumulative']['bytes'] cluster[cur_sample]['cur_thrpt']['total'] = sum(cluster[cur_sample]['cur_thrpt'].values()) cluster[cur_sample]['cur_thrpt']['cache'] = cluster[cur_sample]['cur_thrpt']['ram'] + \ cluster[cur_sample]['cur_thrpt']['disk'] #Preserve the cumulative for next sample set cluster['cumulative']['bytes'] = cluster[cur_sample]['bytes'] #Push to store the data r.rpush(store_q, (cur_sample, dict(cluster[cur_sample]))) del cluster[cur_sample] item_cnt = 1 cur_sample = sample_q.pop(0) if(len(sample_q) > 0) else counters['sample_id'] init_sample_ts = time.time() else: LOG.info("Got new sample ID: %s. Need to wait for current sample(%s) to arrive until pushed out" % (counters['sample_id'], cur_sample)) LOG.info("Adding sample ID to the waiting list.") if counters['sample_id'] not in sample_q: sample_q.append(counters['sample_id']) if cur_sample is None: cur_sample = counters['sample_id'] init_sample_ts = time.time() else: LOG.critical("Device: %s, %s IP: %s" % (counters['device_id'], counters['name'], counters['ip'])) LOG.critical("MFC response doesn't have any counter data. skipping sample: %s" % (counters['sample_id']))
def store_mfc_stats(): from datetime import date from analyticsengine import dbmanager from analyticsengine.dbmanager.mfc.schema import MFC_STATS_TABLE_NAME, MFC_SUMMARY_TABLE_NAME db_connection = dbmanager.connect_cassandra() date_strf = lambda dt: dt.strftime('%m%d%Y') req_interval = int(config.get('collector', 'MFC_REQUEST_FREQUENCY')) while True: data = r.blpop(config.get('constants', 'REDIS_MFC_STORE_QUEUE_KEY')) counters = json.loads(data[1]) """ CF date suffix CF date suffix are calculated based on the timestamp in the response. data can be buffered in queue and may be of different date. Instead of calculating based on current date, its best to stick to the timestamp in the payload as it will then get stored in right CF. This will address the case when queue has buffered data of different date and can recover on a app crash. """ date_str = date_strf(date.fromtimestamp(counters['data']['timestamp'])) DAILY_TABLE_INSERT = "INSERT INTO " + MFC_STATS_TABLE_NAME + date_str + \ """ (mfcid, hostname, ip, ts, type, name, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(ts)s, %(type)s, %(name)s, %(value)s) """ DAILY_SUMMARY_INSERT = "INSERT INTO " + MFC_SUMMARY_TABLE_NAME + date_str + \ """ (mfcid, hostname, ip, sample_id, ts, value) VALUES (%(mfcid)s, %(hostname)s, %(ip)s, %(sample_id)s, %(ts)s, %(value)s) """ pk = glbl_bytes = glbl_ds = glbl_ram = glbl_req = glbl_tier = http_ns = sys_stat = dict( ) """Global stats.""" #Primary Key. pk['mfcid'] = str(counters['device_id']) pk['hostname'] = counters['name'] pk['ip'] = counters['ip'] pk['ts'] = counters['data']['timestamp'] * 1000 #Global Bytes. glbl_bytes.update(pk) glbl_bytes['type'] = 'global' glbl_bytes['name'] = 'bytes' glbl_bytes['value'] = counters['data']['glbl']['bytes'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_bytes, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_bytes) db_connection.execute(DAILY_TABLE_INSERT, glbl_bytes) #Global Disk Space. glbl_ds.update(pk) glbl_ds['type'] = 'global' glbl_ds['name'] = 'disk_space' #Disk space is in MB. Converting to Bytes for k, v in counters['data']['glbl']['disk_space'].items(): counters['data']['glbl']['disk_space'][k] = v * 1024 * 1024 glbl_ds['value'] = counters['data']['glbl']['disk_space'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_ds, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_ds) db_connection.execute(DAILY_TABLE_INSERT, glbl_ds) #Global Ram Cache. glbl_ram.update(pk) glbl_ram['type'] = 'global' glbl_ram['name'] = 'ram_cache' glbl_ram['value'] = counters['data']['glbl']['ram_cache'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_ram, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_ram) db_connection.execute(DAILY_TABLE_INSERT, glbl_ram) #Global Requests. glbl_req.update(pk) glbl_req['type'] = 'global' glbl_req['name'] = 'requests' glbl_req['value'] = counters['data']['glbl']['requests'] #ingest_to_db.apply_async(args=[DailyCounters, session], kwargs=glbl_req, queue='store', # routing_key='store.stats') #DailyMFCCounters.create(**glbl_req) db_connection.execute(DAILY_TABLE_INSERT, glbl_req) #Global Tiers. glbl_tier.update(pk) glbl_tier['type'] = 'global' for tier in counters['data']['glbl']['tiers']: glbl_tier['name'] = tier['provider'] tier.pop('provider') glbl_tier['value'] = tier db_connection.execute(DAILY_TABLE_INSERT, glbl_tier) """Namespace Stats.""" http_ns.update(pk) http_ns['type'] = 'http_ns' for ns in counters['data']['services']['http']['namespaces']: http_ns['name'] = ns['name'] + ':requests' http_ns['value'] = ns['requests'] db_connection.execute(DAILY_TABLE_INSERT, http_ns) http_ns['name'] = ns['name'] + ':bytes' http_ns['value'] = ns['bytes'] db_connection.execute(DAILY_TABLE_INSERT, http_ns) """System Stats.""" sys_stat.update(pk) sys_stat['type'] = 'system' sys_stat['name'] = 'cpu' sys_stat['value'] = counters['data']['system']['cpu'] db_connection.execute(DAILY_TABLE_INSERT, sys_stat) sys_stat['name'] = 'memory' #System Memory is in KB. Converting to Bytes for k, v in counters['data']['system']['memory'].items(): counters['data']['system']['memory'][k] = v * 1024 sys_stat['value'] = counters['data']['system']['memory'] db_connection.execute(DAILY_TABLE_INSERT, sys_stat) """MFC Summary Stats""" sum_stats = dict() sum_stats['mfcid'] = str(counters['device_id']) sum_stats['hostname'] = counters['name'] sum_stats['ip'] = counters['ip'] sum_stats['sample_id'] = counters['sample_id'] sum_stats['ts'] = counters['data']['timestamp'] * 1000 sum_stats['value'] = {} for k, v in counters['data']['glbl']['requests'].items(): sum_stats['value'].update({'req_' + k: str(v)}) for k, v in counters['data']['cur_thrpt'].items(): sum_stats['value'].update({'cur_' + k: str(v / req_interval)}) sum_stats['value'].update({'chr': str(counters['data']['chr'])}) db_connection.execute(DAILY_SUMMARY_INSERT, sum_stats)
def ingest_to_db(model, session, **kwargs): if not session: from cqlengine import connection LOG.info("No c* session found.. creating one.") connection.setup([config.get('cassandra', 'db_host')], config.get('cassandra', 'keyspace')) model.create(**kwargs)