def ingest_file(file,partition,hdfs_root_path,topic,kafka_servers): logger = logging.getLogger('SPOT.INGEST.FLOW.{0}'.format(os.getpid())) try: # get file name and date. file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts)-1] file_date = file_name.split('.')[1] file_date_path = file_date[0:8] file_date_hour = file_date[8:10] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format(hdfs_root_path,file_date_path,file_date_hour) Util.creat_hdfs_folder(hdfs_path,logger) # load to hdfs. hdfs_file = "{0}/{1}".format(hdfs_path,file_name) Util.load_to_hdfs(file,hdfs_file,logger) # create event for workers to process the file. logger.info("Sending file to worker number: {0}".format(partition)) KafkaTopic.SendMessage(hdfs_file,kafka_servers,topic,partition) logger.info("File {0} has been successfully sent to Kafka Topic to: {1}".format(file,topic)) except Exception as err: logger.error("There was a problem, please check the following error message:{0}".format(err.message)) logger.error("Exception: {0}".format(err))
def ingest_file(file, partition, hdfs_root_path, topic, kafka_servers): logger = logging.getLogger('SPOT.INGEST.FLOW.{0}'.format(os.getpid())) try: # get file name and date. file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] file_date = file_name.split('.')[1] file_date_path = file_date[0:8] file_date_hour = file_date[8:10] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format(hdfs_root_path, file_date_path, file_date_hour) Util.creat_hdfs_folder(hdfs_path, logger) # load to hdfs. hdfs_file = "{0}/{1}".format(hdfs_path, file_name) Util.load_to_hdfs(file, hdfs_file, logger) # create event for workers to process the file. logger.info("Sending file to worker number: {0}".format(partition)) KafkaTopic.SendMessage(hdfs_file, kafka_servers, topic, partition) logger.info( "File {0} has been successfully sent to Kafka Topic to: {1}". format(file, topic)) except Exception as err: logger.error( "There was a problem, please check the following error message:{0}" .format(err.message)) logger.error("Exception: {0}".format(err))
def start(self): self._logger.info("Creating Spark Job for topic: {0}".format(self._kafka_consumer.Topic)) # parser parser = self._conf["parser"] #spark conf diver_memory = self._spark_conf["driver_memory"] num_exec = self._spark_conf["spark_exec"] exec_memory = self._spark_conf["spark_executor_memory"] exec_cores = self._spark_conf["spark_executor_cores"] batch_size = self._spark_conf["spark_batch_size"] jar_path = os.path.dirname(os.path.dirname(self._script_path)) # spark job command. spark_job_cmd = ("spark-submit --master yarn " "--driver-memory {0} " "--num-executors {1} " "--conf spark.executor.memory={2} " "--conf spark.executor.cores={3} " "--jars {4}/common/spark-streaming-kafka-0-8-assembly_2.11-2.0.0.jar " "{5}/{6} " "-zk {7} " "-t {8} " "-db {9} " "-dt {10} " "-w {11} " "-bs {12}".format(diver_memory,num_exec,exec_memory,exec_cores,jar_path,self._script_path,parser,self._kafka_consumer.ZookeperServer,self._kafka_consumer.Topic,self._db_name,"proxy",self._processes,batch_size)) # start spark job. Util.execute_cmd(spark_job_cmd,self._logger)
def ingest_file(file, message_size, topic, kafka_servers): logger = logging.getLogger('SPOT.INGEST.PROXY.{0}'.format(os.getpid())) try: message = "" logger.info("Ingesting file: {0} process:{1}".format( file, os.getpid())) with open(file, "rb") as f: for line in f: message += line if len(message) > message_size: KafkaProducer.SendMessage(message, kafka_servers, topic, 0) message = "" #send the last package. KafkaProducer.SendMessage(message, kafka_servers, topic, 0) rm_file = "rm {0}".format(file) Util.execute_cmd(rm_file, logger) logger.info( "File {0} has been successfully sent to Kafka Topic: {1}".format( file, topic)) except Exception as err: logger.error( "There was a problem, please check the following error message:{0}" .format(err.message)) logger.error("Exception: {0}".format(err))
def run(cls): ''' Main command-line entry point. :param cls: The class as implicit first argument. ''' try: args = _parse_args() conf = json.loads(args.config_file.read()) # .........................set up logger Util.get_logger('SPOT', args.log_level) # .........................check kerberos authentication if os.getenv('KRB_AUTH'): kb = Kerberos() kb.authenticate() conf['producer'] = { 'bootstrap_servers': [ '{0}:{1}'.format(conf['kafka']['kafka_server'], conf['kafka']['kafka_port']) ] } conf['file_watcher'] = { 'path': conf['pipelines'][args.type]['collector_path'], 'supported_files': conf['pipelines'][args.type]['supported_files'], 'recursive': True } # .........................migrate configs if not 'local_staging' in conf['pipelines'][args.type].keys(): conf['pipelines'][args.type]['local_staging'] = '/tmp' if 'max_request_size' in conf['kafka'].keys(): conf['producer']['max_request_size'] = conf['kafka'][ 'max_request_size'] if not 'process_opt' in conf['pipelines'][args.type].keys(): conf['pipelines'][args.type]['process_opt'] = '' if 'recursive' in conf['pipelines'][args.type].keys(): conf['file_watcher']['recursive'] = conf['pipelines'][ args.type]['recursive'] collector = cls(args.type, args.topic, args.skip_conversion, **conf) collector.start() except SystemExit: raise except: sys.excepthook(*sys.exc_info()) sys, exit(1)
def test_add_fail(self): res = Util.safe_call(self.dbm.add_driver, 1, {}) self.assertTrue( isinstance(res, DbException), "Expected exception when adding invalid driver, got %s" % res) # check that we can't insert driver with existing id res = Util.safe_call(self.dbm.add_driver, 0, DbManagerTestCase.TestDriver(0)) self.assertTrue(isinstance(res, DbException), "Expected exception, but got %s" % res) self.assertEqual(len(self.dbm), 1, "DB manager size changed")
def test_cmd_all(self): res = Util.safe_call(self.dbm.add_driver, 1, DbManagerTestCase.TestDriver(1)) self.assertTrue(not isinstance(res, Exception), "Exception adding new driver: %s" % res) arg = 1 exp_vals = range(arg, len(self.dbm) + arg) res = Util.safe_call(self.dbm.cmd_all, lambda db: db.echo(arg)) self.assertTrue(not isinstance(res, Exception), "Exception: %s" % res) self.assertTrue(not Util.list_diff(res, exp_vals), "Got %s, expected %s" % (res, exp_vals))
def spark_job(script_file, **kwargs): ''' Run given script file by appling it as a Spark Job. ''' spark_job = 'spark2-submit --master {0}'.format(kwargs.pop('master')) spark_job += ' --deploy-mode {0}'.format(kwargs.pop('deploy_mode')) spark_job += ' --py-files {0}'.format(kwargs.pop('py_files')) if 'driver_memory' in kwargs.keys(): spark_job += ' --driver-memory {0}'.format(kwargs.pop('driver_memory')) if 'spark_exec' in kwargs.keys(): spark_job += ' --num-executors {0}'.format(kwargs.pop('spark_exec')) if 'spark_executor_memory' in kwargs.keys(): spark_job += ' --conf spark.executor.memory={0}'.format( kwargs.pop('spark_executor_memory')) if 'spark_executor_cores' in kwargs.keys(): spark_job += ' --conf spark.executor.cores={0}'.format( kwargs.pop('spark_executor_cores')) spark_job += ' {0}'.format(os.path.abspath(script_file)) if 'spark_batch_size' in kwargs.keys(): spark_job += ' -b {0}'.format(kwargs.pop('spark_batch_size')) spark_job += ' -d {0}'.format(kwargs.pop('database')) if kwargs['group_id'] is not None: spark_job += ' -g {0}'.format(kwargs.pop('group_id')) spark_job += ' -l {0}'.format(kwargs.pop('log_level')) if kwargs['app_name'] is not None: spark_job += ' -n {0}'.format(kwargs.pop('app_name')) spark_job += ' -p {0}'.format(kwargs.pop('partitions')) spark_job += ' -t {0}'.format(kwargs.pop('type')) spark_job += ' --topic {0}'.format(kwargs.pop('topic')) spark_job += ' --zkquorum {0}'.format(kwargs.pop('zkquorum')) if kwargs['redirect_spark_logs'] is not None: spark_job += ' 2>{0}'.format(kwargs.pop('redirect_spark_logs')) try: Util.call(spark_job, True) except Exception as exc: sys.stderr.write('Failed to submit Spark Job!\n') sys.stderr.write('[{0}] {1}\n\n'.format(exc.__class__.__name__, exc.message)) sys.exit(2)
def _create_topic(self): self._logger.info("Creating topic: {0} with {1} parititions".format( self._topic, self._num_of_partitions)) # get script path zk_conf = "{0}:{1}".format(self._zk_server, self._zk_port) create_topic_cmd = "{0}/kafka_topic.sh create {1} {2} {3}".format( os.path.dirname(os.path.abspath(__file__)), self._topic, zk_conf, self._num_of_partitions) # execute create topic cmd Util.execute_cmd(create_topic_cmd, self._logger)
def _ingest_file(new_file, hdfs_root_path, producer, topic): logger = logging.getLogger('SPOT.INGEST.FLOW.{0}'.format(os.getpid())) try: # get file name and date. file_name_parts = new_file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] file_date = file_name.split('.')[1] file_date_path = file_date[0:8] file_date_hour = file_date[8:10] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format(hdfs_root_path, file_date_path, file_date_hour) hdfs_file = "{0}/{1}".format(hdfs_path, file_name) try: if len(hdfs.list_dir(hdfs_path)) == 0: logger.info('creating directory: ' + hdfs_path) hdfs.mkdir(hdfs_path) logger.info('uploading file to hdfs: ' + hdfs_file) result = hdfs.upload_file(hdfs_path, new_file) if not result: logger.error('File failed to upload: ' + hdfs_file) raise HdfsException else: rm_file = "rm {0}".format(new_file) logger.info( "Removing files from local staging: {0}".format(rm_file)) Util.execute_cmd(rm_file, logger) except HdfsException as err: logger.error('Exception: ' + err.exception) logger.info('Check Hdfs Connection settings and server health') except Exception as err: logger.error("There was a problem, Exception: {0}".format(err)) # create event for workers to process the file. # logger.info("Sending file to worker number: {0}".format(partition)) try: producer.SendMessage(hdfs_file, topic) logger.info( "File {0} has been successfully sent to Kafka Topic to: {1}". format(hdfs_file, topic)) except Exception as err: logger.info("File {0} failed to be sent to Kafka Topic to: {1}".format( hdfs_file, topic)) logger.error("Error: {0}".format(err))
def start_collector(type, workers_num, id=None): # generate ingest id ingest_id = str(datetime.datetime.time(datetime.datetime.now())).replace( ":", "_").replace(".", "_") # create logger. logger = Util.get_logger("SPOT.INGEST") # validate the given configuration exists in ingest_conf.json. if not type in master_conf["pipelines"]: logger.error("'{0}' type is not a valid configuration.".format(type)) sys.exit(1) # validate the type is a valid module. if not Util.validate_data_source(master_conf["pipelines"][type]["type"]): logger.error( "'{0}' type is not configured. Please check you ingest conf file". format(master_conf["pipelines"][type]["type"])) sys.exit(1) # validate if kerberos authentication is required. if os.getenv('KRB_AUTH'): kb = Kerberos() kb.authenticate() # kafka server info. logger.info("Initializing kafka instance") k_server = master_conf["kafka"]['kafka_server'] k_port = master_conf["kafka"]['kafka_port'] # required zookeeper info. zk_server = master_conf["kafka"]['zookeper_server'] zk_port = master_conf["kafka"]['zookeper_port'] topic = "SPOT-INGEST-{0}_{1}".format(type, ingest_id) if not id else id kafka = KafkaTopic(topic, k_server, k_port, zk_server, zk_port, workers_num) # create a collector instance based on data source type. logger.info("Starting {0} ingest instance".format(topic)) module = __import__("pipelines.{0}.collector".format( master_conf["pipelines"][type]["type"]), fromlist=['Collector']) # start collector. ingest_collector = module.Collector(master_conf['hdfs_app_path'], kafka, type) ingest_collector.start()
def ingest_file(file, pkt_num, pcap_split_staging, partition, hdfs_root_path, topic, kafka_servers): logger = logging.getLogger('SPOT.INGEST.DNS.{0}'.format(os.getpid())) try: # get file name and date. org_file = file file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] # split file. name = file_name.split('.')[0] split_cmd = "editcap -c {0} {1} {2}/{3}_spot.pcap".format( pkt_num, file, pcap_split_staging, name) logger.info("Splitting file: {0}".format(split_cmd)) Util.execute_cmd(split_cmd, logger) logger.info("Removing file: {0}".format(org_file)) rm_big_file = "rm {0}".format(org_file) Util.execute_cmd(rm_big_file, logger) for currdir, subdir, files in os.walk(pcap_split_staging): for file in files: if file.endswith(".pcap") and "{0}_spot".format(name) in file: # get timestamp from the file name to build hdfs path. file_date = file.split('.')[0] pcap_hour = file_date[-6:-4] pcap_date_path = file_date[-14:-6] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format( hdfs_root_path, pcap_date_path, pcap_hour) # create hdfs path. Util.creat_hdfs_folder(hdfs_path, logger) # load file to hdfs. hadoop_pcap_file = "{0}/{1}".format(hdfs_path, file) Util.load_to_hdfs(os.path.join(currdir, file), hadoop_pcap_file, logger) # create event for workers to process the file. logger.info( "Sending split file to worker number: {0}".format( partition)) KafkaTopic.SendMessage(hadoop_pcap_file, kafka_servers, topic, partition) logger.info( "File {0} has been successfully sent to Kafka Topic to: {1}" .format(file, topic)) except Exception as err: logger.error( "There was a problem, please check the following error message:{0}" .format(err.message)) logger.error("Exception: {0}".format(err))
def test_add(self): self.assertEqual(len(self.dbm), 0, "DB Manager not empty") res = Util.safe_call(self.dbm.add_driver, 0, DbManagerTestCase.TestDriver(0)) self.assertTrue(not isinstance(res, Exception), "Exception adding driver: %s" % res) self.assertEqual(len(self.dbm), 1, "DB Manager is still empty")
def test_cmd_one(self): test_val = 5 res = Util.safe_call(self.dbm.cmd_one, 0, lambda db: db.echo(test_val)) self.assertTrue(not isinstance(res, Exception), "Exception calling method: %s" % res) self.assertEqual(res, test_val, "Got %s, expected %d" % (res, test_val))
def __init__(self): self._logger = Util.get_logger('SPOT.COMMON.KERBEROS') principal, keytab, sasl_mech, security_proto = config.kerberos() if os.getenv('KINITPATH'): self._kinit = os.getenv('KINITPATH') else: self._kinit = "kinit" self._kinitopts = os.getenv('KINITOPTS') self._keytab = "-kt {0}".format(keytab) self._krb_user = principal if self._kinit == None or self._keytab == None or self._krb_user == None: self._logger.error( "Please verify kerberos configuration, some environment variables are missing." ) sys.exit(1) if self._kinitopts is None: self._kinit_cmd = "{0} {1} {2}".format(self._kinit, self._keytab, self._krb_user) else: self._kinit_cmd = "{0} {1} {2} {3}".format(self._kinit, self._kinitopts, self._keytab, self._krb_user)
def _create_topic(self): self._logger.info("Creating topic: {0} with {1} parititions".format(self._topic,self._num_of_partitions)) # Create partitions for the workers. self._partitions = [ TopicPartition(self._topic,p) for p in range(int(self._num_of_partitions))] # create partitioner self._partitioner = RoundRobinPartitioner(self._partitions) # get script path zk_conf = "{0}:{1}".format(self._zk_server,self._zk_port) create_topic_cmd = "{0}/kafka_topic.sh create {1} {2} {3}".format(os.path.dirname(os.path.abspath(__file__)),self._topic,zk_conf,self._num_of_partitions) # execute create topic cmd Util.execute_cmd(create_topic_cmd,self._logger)
def start(self): self._logger.info("Starting PROXY collector") self._watcher.start() try: while True: #self._ingest_files() self._ingest_files_pool() time.sleep(self._ingestion_interval) except KeyboardInterrupt: self._logger.info("Stopping Proxy collector...") Util.remove_kafka_topic(self._kafka_topic.Zookeeper,self._kafka_topic.Topic,self._logger) self._watcher.stop() self._pool.terminate() self._pool.close() self._pool.join()
def start(self): self._logger.info("Starting PROXY collector") self._watcher.start() try: while True: #self._ingest_files() self._ingest_files_pool() time.sleep(self._ingestion_interval) except KeyboardInterrupt: self._logger.info("Stopping Proxy collector...") Util.remove_kafka_topic(self._kafka_topic.Zookeeper,self._kafka_topic.Topic,self._logger) self._watcher.stop() self._pool.terminate() self._pool.close() self._pool.join()
def start(self): self._logger.info("Starting FLOW ingest") self._watcher.start() try: while True: self._ingest_files_pool() time.sleep(self._ingestion_interval) except KeyboardInterrupt: self._logger.info("Stopping FLOW collector...") Util.remove_kafka_topic(self._kafka_topic.Zookeeper,self._kafka_topic.Topic,self._logger) self._watcher.stop() self._pool.terminate() self._pool.close() self._pool.join() SystemExit("Ingest finished...")
def start(self): self._logger.info("Starting DNS ingest") self._watcher.start() try: while True: self._ingest_files_pool() time.sleep(self._ingestion_interval) except KeyboardInterrupt: self._logger.info("Stopping DNS collector...") Util.remove_kafka_topic(self._producer.Zookeeper, self._producer.Topic, self._logger) self._watcher.stop() self._pool.terminate() self._pool.close() self._pool.join() SystemExit("Ingest finished...")
def start_worker(type, topic, id, processes=None): logger = Util.get_logger("SPOT.INGEST.WORKER") # validate the given configuration exists in ingest_conf.json. if not type in WORKER_CONF["pipelines"]: logger.error("'{0}' type is not a valid configuration.".format(type)) sys.exit(1) # validate the type is a valid module. if not Util.validate_data_source(WORKER_CONF["pipelines"][type]["type"]): logger.error("The provided data source {0} is not valid".format(type)) sys.exit(1) # validate if kerberos authentication is requiered. if os.getenv('KRB_AUTH'): kb = Kerberos() kb.authenticate() # create a worker instance based on the data source type. module = __import__("pipelines.{0}.worker".format( WORKER_CONF["pipelines"][type]["type"]), fromlist=['Worker']) # kafka server info. logger.info("Initializing kafka instance") k_server = WORKER_CONF["kafka"]['kafka_server'] k_port = WORKER_CONF["kafka"]['kafka_port'] # required zookeeper info. zk_server = WORKER_CONF["kafka"]['zookeper_server'] zk_port = WORKER_CONF["kafka"]['zookeper_port'] topic = topic # create kafka consumer. kafka_consumer = KafkaConsumer(topic, k_server, k_port, zk_server, zk_port, id) # start worker. db_name = WORKER_CONF['dbname'] app_path = WORKER_CONF['hdfs_app_path'] ingest_worker = module.Worker(db_name, app_path, kafka_consumer, type, processes) ingest_worker.start()
def index(): data = apl.process_todays_claims() if not data.empty: columns = list(data.columns) columns.remove('_id') return render_template("claims.html", date=Util.datetime(), cols=columns, rows=[r for _, r in data.iterrows()]) else: return "No new claims found"
def test_fill_random(self): self._init_db('test') exp_cnt = 5 res = Util.safe_call(self.modb.fill_random, 'check', self.test_temp, exp_cnt) self.assertTrue(not isinstance(res, DbException), "Exception filling DB: %s" % res) cnt = self.modb.active_db['check'].count() self.assertEqual(cnt, exp_cnt + 1, "Expected %d entries, got %d" % (exp_cnt + 1, cnt))
def test_add(self): db_name = 'test' res = Util.safe_call(self.modb.add_db, db_name) self.assertTrue(not isinstance(res, Exception), "Exception adding database: %s" % res) # try adding DB with same name again res = Util.safe_call(self.modb.add_db, db_name) self.assertTrue(isinstance(res, DbException), "Didn't get exception adding same DB name twice") # try accessing database that wasn't added try: self.modb.active_db = 'test1' except Exception as e: self.assertTrue(isinstance(e, DbException), "Expected DB error, but got %s" % e) else: self.assertTrue(False) self.modb.active_db = db_name cnt = self.modb.active_db.count() self.assertEqual(cnt, 1, "Expected 1 table, got %d" % cnt)
def ingest_file(file,message_size,topic,kafka_servers): logger = logging.getLogger('SPOT.INGEST.PROXY.{0}'.format(os.getpid())) try: message = "" logger.info("Ingesting file: {0} process:{1}".format(file,os.getpid())) with open(file,"rb") as f: for line in f: message += line if len(message) > message_size: KafkaTopic.SendMessage(message,kafka_servers,topic,0) message = "" #send the last package. KafkaTopic.SendMessage(message,kafka_servers,topic,0) rm_file = "rm {0}".format(file) Util.execute_cmd(rm_file,logger) logger.info("File {0} has been successfully sent to Kafka Topic: {1}".format(file,topic)) except Exception as err: logger.error("There was a problem, please check the following error message:{0}".format(err.message)) logger.error("Exception: {0}".format(err))
def ingest_file(file,pkt_num,pcap_split_staging, partition,hdfs_root_path,topic,kafka_servers): logger = logging.getLogger('SPOT.INGEST.DNS.{0}'.format(os.getpid())) try: # get file name and date. org_file = file file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts)-1] # split file. name = file_name.split('.')[0] split_cmd = "editcap -c {0} {1} {2}/{3}_spot.pcap".format(pkt_num,file,pcap_split_staging,name) logger.info("Splitting file: {0}".format(split_cmd)) Util.execute_cmd(split_cmd,logger) logger.info("Removing file: {0}".format(org_file)) rm_big_file = "rm {0}".format(org_file) Util.execute_cmd(rm_big_file,logger) for currdir,subdir,files in os.walk(pcap_split_staging): for file in files: if file.endswith(".pcap") and "{0}_spot".format(name) in file: # get timestamp from the file name to build hdfs path. file_date = file.split('.')[0] pcap_hour = file_date[-6:-4] pcap_date_path = file_date[-14:-6] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format(hdfs_root_path,pcap_date_path,pcap_hour) # create hdfs path. Util.creat_hdfs_folder(hdfs_path,logger) # load file to hdfs. hadoop_pcap_file = "{0}/{1}".format(hdfs_path,file) Util.load_to_hdfs(os.path.join(currdir,file),hadoop_pcap_file,logger) # create event for workers to process the file. logger.info( "Sending split file to worker number: {0}".format(partition)) KafkaTopic.SendMessage(hadoop_pcap_file,kafka_servers,topic,partition) logger.info("File {0} has been successfully sent to Kafka Topic to: {1}".format(file,topic)) except Exception as err: logger.error("There was a problem, please check the following error message:{0}".format(err.message)) logger.error("Exception: {0}".format(err))
def start_worker(type,topic,id,processes=None): logger = Util.get_logger("SPOT.INGEST.WORKER") # validate the given configuration exists in ingest_conf.json. if not type in worker_conf["pipelines"]: logger.error("'{0}' type is not a valid configuration.".format(type)); sys.exit(1) # validate the type is a valid module. if not Util.validate_data_source(worker_conf["pipelines"][type]["type"]): logger.error("The provided data source {0} is not valid".format(type));sys.exit(1) # validate if kerberos authentication is requiered. if os.getenv('KRB_AUTH'): kb = Kerberos() kb.authenticate() # create a worker instance based on the data source type. module = __import__("pipelines.{0}.worker".format(worker_conf["pipelines"][type]["type"]),fromlist=['Worker']) # kafka server info. logger.info("Initializing kafka instance") k_server = worker_conf["kafka"]['kafka_server'] k_port = worker_conf["kafka"]['kafka_port'] # required zookeeper info. zk_server = worker_conf["kafka"]['zookeper_server'] zk_port = worker_conf["kafka"]['zookeper_port'] topic = topic # create kafka consumer. kafka_consumer = KafkaConsumer(topic,k_server,k_port,zk_server,zk_port,id) # start worker. db_name = worker_conf['dbname'] app_path = worker_conf['hdfs_app_path'] ingest_worker = module.Worker(db_name,app_path,kafka_consumer,type,processes) ingest_worker.start()
def convert(logfile, tmpdir, opts='', prefix=None): ''' Copy log file to the local staging area. :param logfile: Path of log file. :param tmpdir : Path of local staging area. :param opts : A set of options for the `cp` command. :param prefix : If `prefix` is specified, the file name will begin with that; otherwise, a default `prefix` is used. :returns : Path of log file in local staging area. :rtype : ``str`` ''' logger = logging.getLogger('SPOT.INGEST.PROXY.PROCESS') with tempfile.NamedTemporaryFile(prefix=prefix, dir=tmpdir, delete=False) as fp: command = COMMAND.format(opts, logfile, fp.name) logger.debug('Execute command: {0}'.format(command)) Util.popen(command, raises=True) return fp.name
def _create_topic(self): self._logger.info("Creating topic: {0} with {1} parititions".format( self._topic, self._num_of_partitions)) # Create partitions for the workers. self._partitions = [ TopicPartition(self._topic, p) for p in range(int(self._num_of_partitions)) ] # create partitioner self._partitioner = RoundRobinPartitioner(self._partitions) # get script path zk_conf = "{0}:{1}".format(self._zk_server, self._zk_port) create_topic_cmd = "{0}/kafka_topic.sh create {1} {2} {3}".format( os.path.dirname(os.path.abspath(__file__)), self._topic, zk_conf, self._num_of_partitions) # execute create topic cmd Util.execute_cmd(create_topic_cmd, self._logger)
def start(self): self._logger.info("Creating Spark Job for topic: {0}".format( self._kafka_consumer.Topic)) # parser parser = self._conf["parser"] #spark conf diver_memory = self._spark_conf["driver_memory"] num_exec = self._spark_conf["spark_exec"] exec_memory = self._spark_conf["spark_executor_memory"] exec_cores = self._spark_conf["spark_executor_cores"] batch_size = self._spark_conf["spark_batch_size"] jar_path = os.path.dirname(os.path.dirname(self._script_path)) # spark job command. spark_job_cmd = ( "spark-submit --master yarn " "--driver-memory {0} " "--num-executors {1} " "--conf spark.executor.memory={2} " "--conf spark.executor.cores={3} " "--jars {4}/common/spark-streaming-kafka-0-8-assembly_2.11-2.0.0.jar " "{5}/{6} " "-zk {7} " "-t {8} " "-db {9} " "-dt {10} " "-w {11} " "-bs {12}".format(diver_memory, num_exec, exec_memory, exec_cores, jar_path, self._script_path, parser, self._kafka_consumer.ZookeperServer, self._kafka_consumer.Topic, self._db_name, "proxy", self._processes, batch_size)) # start spark job. Util.execute_cmd(spark_job_cmd, self._logger)
def convert(netflow, tmpdir, opts='', prefix=None): ''' Convert `nfcapd` file to a comma-separated output format. :param netflow : Path of binary file. :param tmpdir : Path of local staging area. :param opts : A set of options for `nfdump` command. :param prefix : If `prefix` is specified, the file name will begin with that; otherwise, a default `prefix` is used. :returns : Path of CSV-converted file. :rtype : ``str`` :raises OSError: If an error occurs while executing the `nfdump` command. ''' logger = logging.getLogger('SPOT.INGEST.FLOW.PROCESS') with tempfile.NamedTemporaryFile(prefix=prefix, dir=tmpdir, delete=False) as fp: command = COMMAND.format(netflow, opts, fp.name) logger.debug('Execute command: {0}'.format(command)) Util.popen(command, raises=True) return fp.name
def _initialize_members(self,db_name,hdfs_app_path,kafka_consumer,conf_type,processes): # get logger instance. self._logger = Util.get_logger('SPOT.INGEST.WRK.PROXY') self._db_name = db_name self._hdfs_app_path = hdfs_app_path self._kafka_consumer = kafka_consumer # read proxy configuration. self._script_path = os.path.dirname(os.path.abspath(__file__)) conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path))) conf = json.loads(open(conf_file).read()) self._spark_conf = conf["spark-streaming"] self._conf = conf["pipelines"][conf_type] self._processes = processes
def _initialize_members(self,db_name,hdfs_app_path,kafka_consumer,conf_type): # get logger instance. self._logger = Util.get_logger('SPOT.INGEST.WRK.FLOW') self._db_name = db_name self._hdfs_app_path = hdfs_app_path # read proxy configuration. self._script_path = os.path.dirname(os.path.abspath(__file__)) conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path))) conf = json.loads(open(conf_file).read()) self._conf = conf["pipelines"][conf_type] self._process_opt = self._conf['process_opt'] self._local_staging = self._conf['local_staging'] self.kafka_consumer = kafka_consumer
def _initialize_members(self, db_name, hdfs_app_path, kafka_consumer, conf_type, processes): # get logger instance. self._logger = Util.get_logger('SPOT.INGEST.WRK.PROXY') self._db_name = db_name self._hdfs_app_path = hdfs_app_path self._kafka_consumer = kafka_consumer # read proxy configuration. self._script_path = os.path.dirname(os.path.abspath(__file__)) conf_file = "{0}/ingest_conf.json".format( os.path.dirname(os.path.dirname(self._script_path))) conf = json.loads(open(conf_file).read()) self._spark_conf = conf["spark-streaming"] self._conf = conf["pipelines"][conf_type] self._processes = processes
def _initialize_members(self, db_name, hdfs_app_path, kafka_consumer, conf_type): # get logger instance. self._logger = Util.get_logger('SPOT.INGEST.WRK.DNS') self._db_name = db_name self._hdfs_app_path = hdfs_app_path # read proxy configuration. self._script_path = os.path.dirname(os.path.abspath(__file__)) conf_file = "{0}/ingest_conf.json".format( os.path.dirname(os.path.dirname(self._script_path))) conf = json.loads(open(conf_file).read()) self._conf = conf["pipelines"][conf_type] self._process_opt = self._conf['process_opt'] self._local_staging = self._conf['local_staging'] self.kafka_consumer = kafka_consumer
def _ingest_file(hdfs_client, new_file, pkt_num, pcap_split_staging, hdfs_root_path, producer, topic): logger = logging.getLogger('SPOT.INGEST.DNS.{0}'.format(os.getpid())) try: # get file name and date. org_file = new_file file_name_parts = new_file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] # split file. name = file_name.split('.')[0] split_cmd = "editcap -c {0} {1} {2}/{3}_spot.pcap".format( pkt_num, new_file, pcap_split_staging, name) logger.info("Splitting file: {0}".format(split_cmd)) Util.execute_cmd(split_cmd, logger) logger.info("Removing file: {0}".format(org_file)) rm_big_file = "rm {0}".format(org_file) Util.execute_cmd(rm_big_file, logger) except Exception as err: logger.error("There was a problem splitting the file: {0}".format( err.message)) logger.error("Exception: {0}".format(err)) for currdir, subdir, files in os.walk(pcap_split_staging): for file in files: if file.endswith(".pcap") and "{0}_spot".format(name) in file: # get timestamp from the file name to build hdfs path. file_date = file.split('.')[0] pcap_hour = file_date[-6:-4] pcap_date_path = file_date[-14:-6] # hdfs path with timestamp. hdfs_path = "{0}/binary/{1}/{2}".format( hdfs_root_path, pcap_date_path, pcap_hour) # create hdfs path. try: if len(hdfs.list_dir(hdfs_path, hdfs_client)) == 0: logger.info('creating directory: ' + hdfs_path) hdfs_client.mkdir(hdfs_path, hdfs_client) # load file to hdfs. hadoop_pcap_file = "{0}/{1}".format(hdfs_path, file) result = hdfs_client.upload_file( hadoop_pcap_file, os.path.join(currdir, file)) if not result: logger.error('File failed to upload: ' + hadoop_pcap_file) raise HdfsException # create event for workers to process the file. logger.info( "Sending split file to Topic: {0}".format(topic)) producer.SendMessage(hadoop_pcap_file, topic) logger.info( "File {0} has been successfully sent to Kafka Topic to: {1}" .format(file, topic)) except HdfsException as err: logger.error('Exception: ' + err.exception) logger.info( 'Check Hdfs Connection settings and server health') except Exception as err: logger.info( "File {0} failed to be sent to Kafka Topic to: {1}". format(new_file, topic)) logger.error("Error: {0}".format(err))
def authenticate(self): Util.execute_cmd(self._kinit_cmd, self._logger) self._logger.info("Kerberos ticket obtained")
def _process_new_file(self, file): # get file from hdfs get_file_cmd = "hadoop fs -get {0} {1}.".format( file, self._local_staging) self._logger.info("Getting file from hdfs: {0}".format(get_file_cmd)) Util.execute_cmd(get_file_cmd, self._logger) # get file name and date file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] binary_hour = file_name_parts[len(file_name_parts) - 2] binary_date_path = file_name_parts[len(file_name_parts) - 3] binary_year = binary_date_path[0:4] binary_month = binary_date_path[4:6] binary_day = binary_date_path[6:8] # build process cmd. process_cmd = "tshark -r {0}{1} {2} > {0}{1}.csv".format( self._local_staging, file_name, self._process_opt) self._logger.info("Processing file: {0}".format(process_cmd)) Util.execute_cmd(process_cmd, self._logger) # create hdfs staging. hdfs_path = "{0}/dns".format(self._hdfs_app_path) staging_timestamp = datetime.datetime.now().strftime('%M%S%f')[:-4] hdfs_staging_path = "{0}/stage/{1}".format(hdfs_path, staging_timestamp) create_staging_cmd = "hadoop fs -mkdir -p {0}".format( hdfs_staging_path) self._logger.info("Creating staging: {0}".format(create_staging_cmd)) Util.execute_cmd(create_staging_cmd, self._logger) # move to stage. mv_to_staging = "hadoop fs -moveFromLocal {0}{1}.csv {2}/.".format( self._local_staging, file_name, hdfs_staging_path) self._logger.info("Moving data to staging: {0}".format(mv_to_staging)) Util.execute_cmd(mv_to_staging, self._logger) #load to avro load_to_avro_cmd = "hive -hiveconf dbname={0} -hiveconf y={1} -hiveconf m={2} -hiveconf d={3} -hiveconf h={4} -hiveconf data_location='{5}' -f pipelines/dns/load_dns_avro_parquet.hql".format( self._db_name, binary_year, binary_month, binary_day, binary_hour, hdfs_staging_path) self._logger.info("Loading data to hive: {0}".format(load_to_avro_cmd)) Util.execute_cmd(load_to_avro_cmd, self._logger) # remove from hdfs staging rm_hdfs_staging_cmd = "hadoop fs -rm -R -skipTrash {0}".format( hdfs_staging_path) self._logger.info( "Removing staging path: {0}".format(rm_hdfs_staging_cmd)) Util.execute_cmd(rm_hdfs_staging_cmd, self._logger) # remove from local staging. rm_local_staging = "rm {0}{1}".format(self._local_staging, file_name) self._logger.info( "Removing files from local staging: {0}".format(rm_local_staging)) Util.execute_cmd(rm_local_staging, self._logger) self._logger.info( "File {0} was successfully processed.".format(file_name))
def _process_new_file(self,file): # get file from hdfs get_file_cmd = "hadoop fs -get {0} {1}.".format(file,self._local_staging) self._logger.info("Getting file from hdfs: {0}".format(get_file_cmd)) Util.execute_cmd(get_file_cmd,self._logger) # get file name and date file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts)-1] flow_date = file_name.split('.')[1] flow_year = flow_date[0:4] flow_month = flow_date[4:6] flow_day = flow_date[6:8] flow_hour = flow_date[8:10] # build process cmd. process_cmd = "nfdump -o csv -r {0}{1} {2} > {0}{1}.csv".format(self._local_staging,file_name,self._process_opt) self._logger.info("Processing file: {0}".format(process_cmd)) Util.execute_cmd(process_cmd,self._logger) # create hdfs staging. hdfs_path = "{0}/flow".format(self._hdfs_app_path) staging_timestamp = datetime.datetime.now().strftime('%M%S%f')[:-4] hdfs_staging_path = "{0}/stage/{1}".format(hdfs_path,staging_timestamp) create_staging_cmd = "hadoop fs -mkdir -p {0}".format(hdfs_staging_path) self._logger.info("Creating staging: {0}".format(create_staging_cmd)) Util.execute_cmd(create_staging_cmd,self._logger) # move to stage. mv_to_staging ="hadoop fs -moveFromLocal {0}{1}.csv {2}/.".format(self._local_staging,file_name,hdfs_staging_path) self._logger.info("Moving data to staging: {0}".format(mv_to_staging)) subprocess.call(mv_to_staging,shell=True) #load to avro load_to_avro_cmd = "hive -hiveconf dbname={0} -hiveconf y={1} -hiveconf m={2} -hiveconf d={3} -hiveconf h={4} -hiveconf data_location='{5}' -f pipelines/flow/load_flow_avro_parquet.hql".format(self._db_name,flow_year,flow_month,flow_day,flow_hour,hdfs_staging_path) self._logger.info( "Loading data to hive: {0}".format(load_to_avro_cmd)) Util.execute_cmd(load_to_avro_cmd,self._logger) # remove from hdfs staging rm_hdfs_staging_cmd = "hadoop fs -rm -R -skipTrash {0}".format(hdfs_staging_path) self._logger.info("Removing staging path: {0}".format(rm_hdfs_staging_cmd)) Util.execute_cmd(rm_hdfs_staging_cmd,self._logger) # remove from local staging. rm_local_staging = "rm {0}{1}".format(self._local_staging,file_name) self._logger.info("Removing files from local staging: {0}".format(rm_local_staging)) Util.execute_cmd(rm_local_staging,self._logger) self._logger.info("File {0} was successfully processed.".format(file_name))
def _process_new_file(self, nf): # get file from hdfs self._logger.info("Getting file from hdfs: {0}".format(nf)) if hdfs.file_exists(nf): hdfs.download_file(nf, self._local_staging) else: self._logger.info("file: {0} not found".format(nf)) # TODO: error handling # get file name and date file_name_parts = file.split('/') file_name = file_name_parts[len(file_name_parts) - 1] binary_hour = file_name_parts[len(file_name_parts) - 2] binary_date_path = file_name_parts[len(file_name_parts) - 3] binary_year = binary_date_path[0:4] binary_month = binary_date_path[4:6] binary_day = binary_date_path[6:8] # build process cmd. process_cmd = "tshark -r {0}{1} {2} > {0}{1}.csv".format( self._local_staging, file_name, self._process_opt) self._logger.info("Processing file: {0}".format(process_cmd)) Util.execute_cmd(process_cmd, self._logger) # create hdfs staging. hdfs_path = "{0}/dns".format(self._hdfs_app_path) staging_timestamp = datetime.datetime.now().strftime('%M%S%f')[:-4] hdfs_staging_path = "{0}/stage/{1}".format(hdfs_path, staging_timestamp) self._logger.info("Creating staging: {0}".format(hdfs_staging_path)) hdfs.mkdir(hdfs_staging_path) # move to stage. local_file = "{0}{1}.csv".format(self._local_staging, file_name) self._logger.info( "Moving data to staging: {0}".format(hdfs_staging_path)) hdfs.upload_file(hdfs_staging_path, local_file) #load to avro drop_table = 'DROP TABLE IF EXISTS {0}.dns_tmp'.format(self._db_name) self._cursor.execute(drop_table) # Create external table create_external = ( "\n" "CREATE EXTERNAL TABLE {0}.dns_tmp (\n" " frame_day STRING,\n" " frame_time STRING,\n" " unix_tstamp BIGINT,\n" " frame_len INT,\n" " ip_src STRING,\n" " ip_dst STRING,\n" " dns_qry_name STRING,\n" " dns_qry_type INT,\n" " dns_qry_class STRING,\n" " dns_qry_rcode INT,\n" " dns_a STRING \n" " )\n" " ROW FORMAT DELIMITED FIELDS TERMINATED BY ','\n" " STORED AS TEXTFILE\n" " LOCATION '{1}'\n" " TBLPROPERTIES ('avro.schema.literal'='{{\n" " \"type\": \"record\"\n" " , \"name\": \"RawDnsRecord\"\n" " , \"namespace\" : \"com.cloudera.accelerators.dns.avro\"\n" " , \"fields\": [\n" " {{\"name\": \"frame_day\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"frame_time\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"unix_tstamp\", \"type\":[\"bigint\", \"null\"]}\n" " , {{\"name\": \"frame_len\", \"type\":[\"int\", \"null\"]}\n" " , {{\"name\": \"ip_src\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"ip_dst\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"dns_qry_name\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"dns_qry_type\", \"type\":[\"int\", \"null\"]}\n" " , {{\"name\": \"dns_qry_class\", \"type\":[\"string\", \"null\"]}\n" " , {{\"name\": \"dns_qry_rcode\", \"type\":[\"int\", \"null\"]}\n" " , {{\"name\": \"dns_a\", \"type\":[\"string\", \"null\"]}\n" " ]\n" "}')\n").format(self._db_name, hdfs_staging_path) self._logger.info( "Creating external table: {0}".format(create_external)) self._cursor.execute(create_external) # Insert data insert_into_table = """ INSERT INTO TABLE {0}.dns PARTITION (y={1}, m={2}, d={3}, h={4) SELECT CONCAT(frame_day , frame_time) as treceived, unix_tstamp, frame_len, ip_dst, ip_src, dns_qry_name, dns_qry_class,dns_qry_type, dns_qry_rcode, dns_a FROM {0}.dns_tmp """.format(self._db_name, binary_year, binary_month, binary_day, binary_hour) self._logger.info("Loading data to {0}: {1}".format( self._db_name, insert_into_table)) self._cursor.execute(insert_into_table) # remove from hdfs staging self._logger.info( "Removing staging path: {0}".format(hdfs_staging_path)) hdfs.delete_folder(hdfs_staging_path) # remove from local staging. rm_local_staging = "rm {0}{1}".format(self._local_staging, file_name) self._logger.info( "Removing files from local staging: {0}".format(rm_local_staging)) Util.execute_cmd(rm_local_staging, self._logger) self._logger.info( "File {0} was successfully processed.".format(file_name))
def _process_new_file(self, nf): # get file name and date file_name_parts = nf.split('/') file_name = file_name_parts[len(file_name_parts) - 1] nf_path = nf.rstrip(file_name) flow_date = file_name.split('.')[1] flow_year = flow_date[0:4] flow_month = flow_date[4:6] flow_day = flow_date[6:8] flow_hour = flow_date[8:10] # get file from hdfs if hdfs.file_exists(nf_path, file_name): self._logger.info("Getting file from hdfs: {0}".format(nf)) hdfs.download_file(nf, self._local_staging) else: self._logger.info("file: {0} not found".format(nf)) # TODO: error handling # build process cmd. sf = "{0}{1}.csv".format(self._local_staging, file_name) process_cmd = "nfdump -o csv -r {0}{1} {2} > {3}".format( self._local_staging, file_name, self._process_opt, sf) self._logger.info("Processing file: {0}".format(process_cmd)) Util.execute_cmd(process_cmd, self._logger) # create hdfs staging. hdfs_path = "{0}/flow".format(self._hdfs_app_path) staging_timestamp = datetime.datetime.now().strftime('%M%S%f')[:-4] hdfs_staging_path = "{0}/stage/{1}".format(hdfs_path, staging_timestamp) self._logger.info("Creating staging: {0}".format(hdfs_staging_path)) hdfs.mkdir(hdfs_staging_path) # move to stage. local_file = "{0}{1}.csv".format(self._local_staging, file_name) self._logger.info( "Moving data to staging: {0}".format(hdfs_staging_path)) hdfs.upload_file(hdfs_staging_path, local_file) # load with impyla drop_table = "DROP TABLE IF EXISTS {0}.flow_tmp".format(self._db_name) self._logger.info("Dropping temp table: {0}".format(drop_table)) self._cursor.execute_query(drop_table) create_external = ( "\n" "CREATE EXTERNAL TABLE {0}.flow_tmp (\n" " treceived STRING,\n" " tryear INT,\n" " trmonth INT,\n" " trday INT,\n" " trhour INT,\n" " trminute INT,\n" " trsec INT,\n" " tdur FLOAT,\n" " sip STRING,\n" " dip STRING,\n" " sport INT,\n" " dport INT,\n" " proto STRING,\n" " flag STRING,\n" " fwd INT,\n" " stos INT,\n" " ipkt BIGINT,\n" " ibyt BIGINT,\n" " opkt BIGINT,\n" " obyt BIGINT,\n" " input INT,\n" " output INT,\n" " sas INT,\n" " das INT,\n" " dtos INT,\n" " dir INT,\n" " rip STRING\n" " )\n" " ROW FORMAT DELIMITED FIELDS TERMINATED BY ','\n" " STORED AS TEXTFILE\n" " LOCATION '{1}'\n" " TBLPROPERTIES ('avro.schema.literal'='{{\n" " \"type\": \"record\"\n" " , \"name\": \"RawFlowRecord\"\n" " , \"namespace\" : \"com.cloudera.accelerators.flows.avro\"\n" " , \"fields\": [\n" " {{\"name\": \"treceived\", \"type\":[\"string\", \"null\"]}}\n" " , {{\"name\": \"tryear\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"trmonth\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"trday\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"trhour\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"trminute\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"trsec\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"tdur\", \"type\":[\"float\", \"null\"]}}\n" " , {{\"name\": \"sip\", \"type\":[\"string\", \"null\"]}}\n" " , {{\"name\": \"sport\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"dip\", \"type\":[\"string\", \"null\"]}}\n" " , {{\"name\": \"dport\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"proto\", \"type\":[\"string\", \"null\"]}}\n" " , {{\"name\": \"flag\", \"type\":[\"string\", \"null\"]}}\n" " , {{\"name\": \"fwd\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"stos\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"ipkt\", \"type\":[\"bigint\", \"null\"]}}\n" " , {{\"name\": \"ibyt\", \"type\":[\"bigint\", \"null\"]}}\n" " , {{\"name\": \"opkt\", \"type\":[\"bigint\", \"null\"]}}\n" " , {{\"name\": \"obyt\", \"type\":[\"bigint\", \"null\"]}}\n" " , {{\"name\": \"input\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"output\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"sas\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"das\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"dtos\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"dir\", \"type\":[\"int\", \"null\"]}}\n" " , {{\"name\": \"rip\", \"type\":[\"string\", \"null\"]}}\n" " ]\n" "}}')\n").format(self._db_name, hdfs_staging_path) self._logger.info( "Creating external table: {0}".format(create_external)) self._cursor.execute_query(create_external) insert_into_table = """ INSERT INTO TABLE {0}.flow PARTITION (y={1}, m={2}, d={3}, h={4}) SELECT treceived, unix_timestamp(treceived) AS unix_tstamp, tryear, trmonth, trday, trhour, trminute, trsec, tdur, sip, dip, sport, dport, proto, flag, fwd, stos, ipkt, ibyt, opkt, obyt, input, output, sas, das, dtos, dir, rip FROM {0}.flow_tmp """.format(self._db_name, flow_year, flow_month, flow_day, flow_hour) self._logger.info("Loading data to {0}: {1}".format( self._db_name, insert_into_table)) self._cursor.execute_query(insert_into_table) # remove from hdfs staging self._logger.info( "Removing staging path: {0}".format(hdfs_staging_path)) hdfs.delete_folder(hdfs_staging_path) # remove from local staging. rm_local_staging = "rm {0}{1}".format(self._local_staging, file_name) self._logger.info( "Removing files from local staging: {0}".format(rm_local_staging)) Util.execute_cmd(rm_local_staging, self._logger) rm_local_staging = "rm {0}".format(sf) self._logger.info( "Removing files from local staging: {0}".format(rm_local_staging)) Util.execute_cmd(rm_local_staging, self._logger) self._logger.info( "File {0} was successfully processed.".format(file_name))