def indexpattern_generate(start, end, raw=False, allraw=False):
"""Function to return the proper index pattern for queries to elasticsearch on gracc.opensciencegrid.org. This improves performance by not just using a general index pattern unless absolutely necessary.
This will especially help with reports, for example.
This function assumes that the date being passed in has been split into a list with [yyyy,mm,dd] format. This gets tested and cleaned up in the called dateparse function.
"""
if allraw:
return 'gracc.osg.raw-*'
if not raw and not allraw: # Default
return 'gracc.osg.summary'
# Raw is True, allraw is False
t = TimeUtils()
basepattern = 'gracc.osg.raw-'
if start.year == end.year:
basepattern += '{0}.'.format(str(start.year))
if start.month == end.month:
if len(str(start.month)) == 1:
add = '0{0}'.format(str(start.month))
else:
add = str(start.month)
basepattern += '{0}'.format(add)
else:
basepattern += '*'
else:
basepattern += '*'
return basepattern
def __init__(self, config_file, flush_cache=False, verbose=False):
self.max_forecasts = []
self.lines_processed = 0
self.delimiter = '\t'
with open(config_file) as conf_file:
config = json.load(conf_file)
self.error_report = ErrorReport()
self.time_utils = TimeUtils()
self.verbose = verbose
self.geo_precision = config['log_processor']['geo_precision']
self.geoip_db = open_database(config['geoip_db']['file'])
self.weather_api = OpenWeatherMap(api_key=config['weather_api']['api_key'],
units=config['weather_api']['units'])
self.weather_cache = WeatherCache(host=config['cache']['host'],
port=config['cache']['port'],
redis_key_expiry_secs=config['cache']['key_expiry_secs']
)
if flush_cache:
self.weather_cache.flush()
def initial_setup():
time_utils = TimeUtils()
logging.basicConfig(filename=("log_" + str(time_utils.get_time())), level=logging.INFO)
