def __init__(self,
fields=None,
qt=None,
q_id=None,
query=None,
sensors=None):
self.query = None # string: query formatted for stenoapi
self.state = None # string: one of (RECEIVING, RECEIVED, MERGE, SUCCESS, ERROR, FAIL)
self._id = None # a hash of the normalized query string, uniquely identifies this query to prevent duplicates
self.events = [
] # a sorted list of events including errors... TODO FIX ME
self.sensors = [
] # limit the query to those sensors (empty means execute on every sensors)
if qt:
self._detupify(qt)
if q_id and is_str(q_id) and len(q_id) >= 32:
self.load(q_id=q_id)
if query and is_str(query):
self.query = query
if sensors:
self.sensors = sensors
if fields and isinstance(fields, dict):
self._build_query(fields)
def job_path_for_id(
q_id,
path=None
): # ensure canonical naming - everything uses this function
parts = (Config.get('SPOOL_DIR'), q_id)
if path:
parts += (path, ) if is_str(path) else path
return os.path.join(*parts)
def find(cls, fields):
r = Config.redis()
if not r:
return []
results = []
ids = Query.get_unexpired()
for i in ids:
q = Query(q_id=i)
if not q.query:
# sometimes query meta data is incomplete, usually when I'm break^H^H^H^H^Htesting.
continue
for k, v in fields.items():
if k in ('after-ago', 'after', 'before-ago', 'before'):
dur = parse_duration(v)
if dur:
v = (datetime.utcnow() - dur)
else:
v = inputs.datetime_from_iso8601(v)
pass
if (q.queried < v) and k in ('after-ago', 'after'):
q = None
break
elif (q.queried > v) and k in ('before-ago', 'before'):
q = None
break
pass
elif k in ('sensors', ):
if frozenset(q.sensors) != frozenset(v):
q = None
break
elif k in ('limit-packets', 'limit-bytes'):
continue
elif k not in q.query:
Config.logger.info("Skipping: {} - {}".format(q.query, k))
q = None
break
else:
if is_sequence(v) and v != [
vi for vi in v if q.query.find(vi) >= 0
]:
Config.logger.info("Skipping: {} - {}".format(
q.query, v))
q = None
break
elif is_str(v) and v not in q.query:
Config.logger.info("Skipping: {} - {}".format(
q.query, v))
q = None
break
if q:
results.append(q.json())
return results
def get_stats(selected_sensors=None):
""" return a dictionary of { sensorname: {stats} }
NOTE: runs immediately not - used when we _need_ a result
"""
Config.logger.debug("Get Stats: {}".format(selected_sensors))
if is_str(selected_sensors):
selected_sensors = (selected_sensors, )
datas = {}
global _INSTANCES
for instance in _INSTANCES:
if ((selected_sensors is not None)
and (instance['sensor'] not in selected_sensors)):
continue
Config.logger.debug("query instance: {}".format(instance['sensor']))
url = "https://{}:{}/debug/stats".format(instance['host'],
instance['port'])
data = None
try:
rq = requests.get(url,
cert=(instance['cert'], instance['key']),
verify=instance['ca'],
timeout=STAT_TIMEOUT)
if rq.status_code != requests.codes.ok:
Config.logger.error("stats failed({}): {} {}".format(
rq.status_code, instance['sensor'], rq.text))
continue
except requests.exceptions.ConnectTimeout as ex:
Config.logger.error(
"Stats: {}:{} Connection timeout after {} seconds".format(
instance['host'], instance['port'], STAT_TIMEOUT))
data = "Connection Timeout"
except requests.exceptions.ReadTimeout as ex:
Config.logger.error(
"Stats: {}:{} Didn't provide stats in {} seconds".format(
instance['host'], instance['port'], STAT_TIMEOUT))
data = "Read Timeout"
except requests.exceptions.ConnectionError as ex:
Config.logger.error("Stats: {}:{} Connection Error? {}".format(
instance['host'], instance['port'], str(ex)))
raise ex
except requests.exceptions.SSLError as ex:
Config.logger.error(
"Stats: {}:{} SSL Error - check conf {}".format(
instance['host'], instance['port'], str(ex)))
raise ex
else:
# Request Succeeded
Config.logger.debug("response code: {}".format(rq.status_code))
lines = rq.text.split('\n')
data = instance.get('stats')
for line in lines:
if line.strip():
k, v = line.split()
data[k] = int(v)
if 'oldest_timestamp' in data:
ot = data['oldest_timestamp']
dt = datetime.utcfromtimestamp(0) + timedelta(microseconds=ot /
1000)
data['oldest_timestamp'] = dt.strftime(ISOFORMAT)
if data.get('indexfile_current_reads') == 0:
instance['idle'] = datetime.utcnow()
datas[instance['sensor']] = data
return datas