def _initialize_members(self, hdfs_app_path, kafka_topic, conf_type):
# getting parameters.
self._logger = logging.getLogger('ONI.INGEST.FLOW')
self._hdfs_app_path = hdfs_app_path
self._kafka_topic = kafka_topic
# get script path
self._script_path = os.path.dirname(os.path.abspath(__file__))
# read flow configuration.
conf_file = "{0}/ingest_conf.json".format(
os.path.dirname(os.path.dirname(self._script_path)))
conf = json.loads(open(conf_file).read())
self._conf = conf["pipelines"][conf_type]
# set configuration.
self._collector_path = self._conf['collector_path']
self._dsource = 'flow'
self._hdfs_root_path = "{0}/{1}".format(hdfs_app_path, self._dsource)
# initialize message broker client.
self.kafka_topic = kafka_topic
# create collector watcher
self._watcher = Util.create_watcher(self._collector_path,
NewFileEvent(self), self._logger)
def _initialize_members(self,hdfs_app_path,kafka_topic,conf_type):
# getting parameters.
self._logger = logging.getLogger('ONI.INGEST.DNS')
self._hdfs_app_path = hdfs_app_path
self._kafka_topic = kafka_topic
# get script path
self._script_path = os.path.dirname(os.path.abspath(__file__))
# read dns configuration.
conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
conf = json.loads(open(conf_file).read())
self._conf = conf["pipelines"][conf_type]
# set configuration.
self._collector_path = self._conf['collector_path']
self._dsource = 'dns'
self._hdfs_root_path = "{0}/{1}".format(hdfs_app_path, self._dsource)
# set configuration.
self._pkt_num = self._conf['pkt_num']
self._pcap_split_staging = self._conf['pcap_split_staging']
# initialize message broker client.
self.kafka_topic = kafka_topic
# create collector watcher
self._watcher = Util.create_watcher(self._collector_path,NewFileEvent(self),self._logger)
def _initialize_members(self,hdfs_app_path,kafka_topic,conf_type):
# getting parameters.
self._logger = logging.getLogger('ONI.INGEST.PROXY')
self._hdfs_app_path = hdfs_app_path
self._kafka_topic = kafka_topic
# get script path
self._script_path = os.path.dirname(os.path.abspath(__file__))
# read proxy configuration.
conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
conf = json.loads(open(conf_file).read())
self._message_size = conf["kafka"]["message_size"]
self._conf = conf["pipelines"][conf_type]
# get collector path.
self._collector_path = self._conf['collector_path']
# create collector watcher
self._watcher = Util.create_watcher(self._collector_path,NewFileEvent(self),self._logger)
def _initialize_members(self, hdfs_app_path, kafka_topic, conf_type):
# getting parameters.
self._logger = logging.getLogger('ONI.INGEST.PROXY')
self._hdfs_app_path = hdfs_app_path
self._kafka_topic = kafka_topic
# get script path
self._script_path = os.path.dirname(os.path.abspath(__file__))
# read proxy configuration.
conf_file = "{0}/ingest_conf.json".format(
os.path.dirname(os.path.dirname(self._script_path)))
conf = json.loads(open(conf_file).read())
self._message_size = conf["kafka"]["message_size"]
self._conf = conf["pipelines"][conf_type]
# get collector path.
self._collector_path = self._conf['collector_path']
# create collector watcher
self._watcher = Util.create_watcher(self._collector_path,
NewFileEvent(self), self._logger)
