def __init__(self, domainType, doStats=0):
self.start = datetime.now()
configFile = "./config.cfg"
config = configparser.ConfigParser()
config.read(configFile)
self.packetSize = config.getint("domainParser", "packetSize")
logger.info("domainReader Starting up")
# clientName = 'parser_' + str(os.getpid())
clientName = "parser_0000"
self.q = mqlightQueue(config, clientName)
self.regex = re.compile(config.get(domainType, "regex"))
self.path = config.get(domainType, "path")
self.doStats = doStats
self.stats = {"domains": 0, "startTime": 0, "endTime": 0, "runningSeconds": 0, "avg": []}
def __init__(self):
configFile = './config.cfg'
config = configparser.ConfigParser()
config.read(configFile)
self.packetSize = config.getint('domainParser','packetSize')
logger.info("Packet size is: %s" % self.packetSize)
# clientName = 'resolver_' + str(os.getpid())
clientName = 'resolver_000'
self.q = mqlightQueue(config,clientName)
while not self.q.ready:
logger.info("RESOLVER: Not ready yet")
time.sleep(1)
self.doStats = 0
self.stats = {
'domains' : 0,
'startTime': 0,
'endTime' : 0,
'runningSeconds': 0,
'avg': []
}
def __init__(self, domainType, doStats=0):
self.start = datetime.now()
configFile = './config.cfg'
config = configparser.ConfigParser()
config.read(configFile)
self.packetSize = config.getint('domainParser', 'packetSize')
logger.info("domainReader Starting up")
# clientName = 'parser_' + str(os.getpid())
clientName = 'parser_0000'
self.q = mqlightQueue(config, clientName)
self.regex = re.compile(config.get(domainType, 'regex'))
self.path = config.get(domainType, 'path')
self.doStats = doStats
self.stats = {
'domains': 0,
'startTime': 0,
'endTime': 0,
'runningSeconds': 0,
'avg': []
}
def __init__(self,indexName="domain-final"):
configFile = './config.cfg'
config = configparser.ConfigParser()
config.read(configFile)
self.packetSize = config.getint('domainParser','packetSize')
clientName = 'consumer_000'
self.q = mqlightQueue(config,clientName)
while not self.q.ready:
logging.info("Consumer: Not ready yet")
time.sleep(1)
my_config = {
'user': config.get('mysql','user'),
'password': config.get('mysql','password'),
'host': config.get('mysql','host'),
'database': config.get('mysql','database')
}
self.es = elasticsearch.Elasticsearch([{'host':config.get('elasticsearch','host')}])
es_log = logging.getLogger("elasticsearch")
es_log.setLevel(logging.CRITICAL)
es_log.disabled=True
logging.getLogger("elasticsearch.trace").setLevel(logging.CRITICAL)
self.sql = mysql.connector.connect(**my_config)
self.cursor = self.sql.cursor()
self.query = ("SELECT ip from ip_address_unique WHERE ip = %(int_ip)s LIMIT 1")
self.doStats = 0
self.index = indexName
self.doStats = 0
self.stats = {
'domains' : 0,
'startTime' : 0,
'endTime' : 0,
'runningSeconds': 0,
'avg': []
}
