def Main(host_name, config_name, group_name, previous_time, current_time): reload(parsers) # for d0uble - he wants to reload parsing functions uuid = hashlib.md5("%s%s%s%i%i" %(host_name, config_name, group_name, previous_time, current_time)).hexdigest()[:10] logger = ParsingLogger(uuid) logger.info("Start parsing: %s %s %s %i %i" %(host_name, config_name, group_name, previous_time, current_time)) conf = ParsingConfigurator(config_name) # Construct parser function parser = PARSERS.get(conf.parser, None) if parser is None: logger.error('No properly parser available') return "failed; No parser" # Construct Distributed Storage ds = DistributedStorageFactory(**conf.ds) # Get Distributed storage if ds is None: logger.error('Failed to init distributed storage like MongoRS') return 'failed; DS init Error' if not ds.connect('combaine_mid/%s' % config_name): # CHECK NAME OF COLLECTION!!!! logger.error('Cannot connect to distributed storage like MongoRS') return 'failed; Connect to DS' # Construct Data Fetcher df = FetcherFactory(**conf.df) # Get DataFetcher if df is None: logger.error('%s Failed to init datafetcher' % uuid) return 'failed; Failed to init DF' # Construct aggregators aggs = [AggregatorFactory(**agg_config) for agg_config in conf.aggregators] #fetch data data = df.getData(host_name, (previous_time, current_time)) if not data: logger.warning('%s Empty data from datafetcher' % uuid) return 'failed; Empty data from DF' handle_data = (l for l in parser(data) if df.filter(l)) handle_data = [l for l in handle_data if l is not None] if len(handle_data) == 0: logger.info("Zero size of handling data list after parsing and filter") return 'failed; Zero size of handling data list after parsing and filter' # TBD wrap in separate fucntion -> if any(_agg.agg_type == TYPES.index("RAW") for _agg in aggs): db = DataGridFactory(**conf.db) # Get DataGrid if db is None: logger.error('Failed to init local databse') return 'failed; Failed to init DG' [_agg.set_datagrid_backend(db) for _agg in aggs if _agg.agg_type == TYPES.index("RAW")] tablename = ''.join(group_name[:30]) + hashlib.md5('%s_%s_%s' % (config_name, group_name, host_name)).hexdigest() if not db.putData(handle_data , tablename): logger.warning('Empty data for localdb') return 'failed; No data for local db' # TBD end of wrap if any(_agg.agg_type == TYPES.index("PROCESSED") for _agg in aggs): [_agg.set_data(handle_data) for _agg in aggs if _agg.agg_type == TYPES.index("PROCESSED")] res = itertools.chain([_agg.aggregate((previous_time, current_time)) for _agg in aggs]) logger.debug("Send data to storage: %s" % [ds.insert("%(host)s;%(conf)s;%(time)s;%(etime)s;%(aggname)s" % {\ 'host' : host_name.replace('.','_').replace('-','_'),\ 'conf' : config_name,\ 'time' : previous_time,\ 'etime' : current_time,\ 'aggname' : l[0]}, l[1]) for l in res]) ds.close() logger.info('Parsing has finished successfully') return 'success'