def do_list_usage_deletes(request):
filter_args = {}
if 'instance' in request.GET:
uuid = request.GET['instance']
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
filter_args['instance'] = uuid
if len(filter_args) > 0:
deletes = models.InstanceDeletes.objects.filter(**filter_args)
else:
deletes = models.InstanceDeletes.objects.all()
results = [["UUID", "Launched At", "Deleted At"]]
for delete in deletes:
launched = None
if delete.launched_at:
launched = str(dt.dt_from_decimal(delete.launched_at))
deleted = None
if delete.deleted_at:
deleted = str(dt.dt_from_decimal(delete.deleted_at))
results.append([delete.instance, launched, deleted])
return rsp(json.dumps(results))
def do_list_usage_exists(request):
filter_args = {}
if 'instance' in request.GET:
uuid = request.GET['instance']
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
filter_args['instance'] = uuid
if len(filter_args) > 0:
exists = models.InstanceExists.objects.filter(**filter_args)
else:
exists = models.InstanceExists.objects.all()
results = [["UUID", "Launched At", "Deleted At", "Instance Type Id",
"Message ID", "Status"]]
for exist in exists:
launched = None
if exist.launched_at:
launched = str(dt.dt_from_decimal(exist.launched_at))
deleted = None
if exist.deleted_at:
deleted = str(dt.dt_from_decimal(exist.deleted_at))
results.append([exist.instance, launched, deleted,
exist.instance_type_id, exist.message_id,
exist.status])
return rsp(json.dumps(results))
def do_uuid(request):
uuid = str(request.GET['uuid'])
service = str(request.GET.get('service', 'nova'))
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
model = _model_factory(service)
result = []
filters = {}
if service == 'nova' or service == 'generic':
filters = {'instance': uuid}
if service == 'glance':
filters = {'uuid': uuid}
_add_when_filters(request, filters)
related = model_search(request,
model,
filters,
related=True,
order_by='when')
for event in related:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
result = event.search_results(result, when, routing_key_status)
return rsp(json.dumps(result))
def do_watch(request, deployment_id):
service = str(request.GET.get('service', 'nova'))
model = _model_factory(service)
deployment_id = int(deployment_id)
since = request.GET.get('since')
event_name = request.GET.get('event_name')
deployment_map = {}
for d in get_deployments():
deployment_map[d.id] = d
events = get_event_names()
max_event_width = max([len(event['event']) for event in events])
base_events = model.order_by('when')
if deployment_id > 0:
base_events = base_events.filter(deployment=deployment_id)
if event_name:
base_events = base_events.filter(event=event_name)
# Ok, this may seem a little wonky, but I'm clamping the
# query time to the closest second. The implication is we
# may not return the absolute latest data (which will have
# to wait for the next query). The upside of doing this
# is we can effectively cache the responses. So, with a
# caching proxy server we can service a lot more clients
# without having to worry about microsecond differences
# causing cache misses.
now = datetime.datetime.utcnow()
now = now.replace(microsecond=0) # clamp it down.
dec_now = dt.dt_to_decimal(now)
if since:
since = decimal.Decimal(since)
else:
since = now - datetime.timedelta(seconds=2)
since = dt.dt_to_decimal(since)
base_events = base_events.filter(when__gt=since)
events = base_events.filter(when__lte=dec_now)
c = [10, 1, 15, 20, max_event_width, 36]
results = []
for raw in events:
uuid = raw.uuid
if not uuid:
uuid = "-"
typ = routing_key_type(raw.routing_key)
when = dt.dt_from_decimal(raw.when)
results.append([
raw.id, typ,
str(when.date()),
str(when.time()), deployment_map[raw.deployment.id].name,
raw.event, uuid
])
results_json = json.dumps([c, results, str(dec_now)])
return rsp(results_json)
def do_list_usage_launches(request):
filter_args = {}
if 'instance' in request.GET:
uuid = request.GET['instance']
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
filter_args['instance'] = uuid
model = models.InstanceUsage.objects
if len(filter_args) > 0:
launches = model_search(request, model, filter_args)
else:
launches = model_search(request, model, None)
results = [[
"UUID", "Launched At", "Instance Type Id", "Instance Flavor Id"
]]
for launch in launches:
launched = None
if launch.launched_at:
launched = str(dt.dt_from_decimal(launch.launched_at))
results.append([
launch.instance, launched, launch.instance_type_id,
launch.instance_flavor_id
])
return rsp(json.dumps(results))
def do_list_usage_launches(request):
filter_args = {}
if 'instance' in request.GET:
uuid = request.GET['instance']
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
filter_args['instance'] = uuid
model = models.InstanceUsage.objects
if len(filter_args) > 0:
launches = model_search(request, model, filter_args)
else:
launches = model_search(request, model, None)
results = [["UUID", "Launched At", "Instance Type Id",
"Instance Flavor Id"]]
for launch in launches:
launched = None
if launch.launched_at:
launched = str(dt.dt_from_decimal(launch.launched_at))
results.append([launch.instance, launched, launch.instance_type_id,
launch.instance_flavor_id])
return rsp(json.dumps(results))
def do_show(request, event_id):
event_id = int(event_id)
results = []
event = None
try:
event = models.RawData.objects.get(id=event_id)
except models.RawData.ObjectDoesNotExist:
return results
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
results.append(["When", str(when)])
results.append(["Deployment", event.deployment.name])
results.append(["Category", event.routing_key])
results.append(["Publisher", event.publisher])
results.append(["State", event.state])
results.append(["Event", event.event])
results.append(["Service", event.service])
results.append(["Host", event.host])
results.append(["UUID", event.instance])
results.append(["Req ID", event.request_id])
final = [results, ]
j = json.loads(event.json)
final.append(json.dumps(j, indent=2))
final.append(event.instance)
return rsp(json.dumps(final))
def do_jsonreports_search(request):
model = models.JsonReport.objects
filters = {}
for filter, value in request.GET.iteritems():
filters[filter + '__exact'] = value
try:
reports = model_search(request, model, filters)
except FieldError:
args = request.GET.keys()
args.sort()
return error_response(
400, 'Bad Request', "The requested fields do not exist for "
"the corresponding object: %s. Note: The field names of database "
"are case-sensitive." % ', '.join(args))
results = [['Id', 'Start', 'End', 'Created', 'Name', 'Version']]
for report in reports:
results.append([report.id,
datetime.datetime.strftime(
report.period_start, UTC_FORMAT),
datetime.datetime.strftime(
report.period_end, UTC_FORMAT),
datetime.datetime.strftime(
dt.dt_from_decimal(report.created),
UTC_FORMAT),
report.name,
report.version])
return rsp(json.dumps(results))
def do_watch(request, deployment_id):
service = str(request.GET.get('service', 'nova'))
model = _model_factory(service)
deployment_id = int(deployment_id)
since = request.GET.get('since')
event_name = request.GET.get('event_name')
deployment_map = {}
for d in get_deployments():
deployment_map[d.id] = d
events = get_event_names()
max_event_width = max([len(event['event']) for event in events])
base_events = model.order_by('when')
if deployment_id > 0:
base_events = base_events.filter(deployment=deployment_id)
if event_name:
base_events = base_events.filter(event=event_name)
# Ok, this may seem a little wonky, but I'm clamping the
# query time to the closest second. The implication is we
# may not return the absolute latest data (which will have
# to wait for the next query). The upside of doing this
# is we can effectively cache the responses. So, with a
# caching proxy server we can service a lot more clients
# without having to worry about microsecond differences
# causing cache misses.
now = datetime.datetime.utcnow()
now = now.replace(microsecond=0) # clamp it down.
dec_now = dt.dt_to_decimal(now)
if since:
since = decimal.Decimal(since)
else:
since = now - datetime.timedelta(seconds=2)
since = dt.dt_to_decimal(since)
base_events = base_events.filter(when__gt=since)
events = base_events.filter(when__lte=dec_now)
c = [10, 1, 15, 20, max_event_width, 36]
results = []
for raw in events:
uuid = raw.uuid
if not uuid:
uuid = "-"
typ = routing_key_type(raw.routing_key)
when = dt.dt_from_decimal(raw.when)
results.append([raw.id, typ,
str(when.date()), str(when.time()),
deployment_map[raw.deployment.id].name,
raw.event,
uuid])
results_json = json.dumps([c, results, str(dec_now)])
return rsp(results_json)
def do_uuid(request):
uuid = str(request.GET['uuid'])
related = models.RawData.objects.select_related(). \
filter(instance=uuid).order_by('when')
results = []
results.append(["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"])
for e in related:
when = dt.dt_from_decimal(e.when)
results.append([e.id, routing_key_type(e.routing_key), str(when),
e.deployment.name, e.event,
e.host,
e.state, e.old_state, e.old_task])
return rsp(results)
def do_request(request):
request_id = request.GET['request_id']
events = models.RawData.objects.filter(request_id=request_id) \
.order_by('when')
results = []
results.append(["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"])
for e in events:
when = dt.dt_from_decimal(e.when)
results.append([e.id, routing_key_type(e.routing_key), str(when),
e.deployment.name, e.event,
e.host, e.state,
e.old_state, e.old_task])
return rsp(results)
def do_list_usage_exists(request):
filter_args = {}
if "instance" in request.GET:
filter_args["instance"] = request.GET["instance"]
if len(filter_args) > 0:
exists = models.InstanceExists.objects.filter(**filter_args)
else:
exists = models.InstanceExists.objects.all()
results = []
results.append(["UUID", "Launched At", "Deleted At", "Instance Type Id", "Message ID", "Status"])
for exist in exists:
launched = None
if exist.launched_at:
launched = str(dt.dt_from_decimal(exist.launched_at))
deleted = None
if exist.deleted_at:
deleted = str(dt.dt_from_decimal(exist.deleted_at))
results.append([exist.instance, launched, deleted, exist.instance_type_id, exist.message_id, exist.status])
return rsp(results)
def append_generic_raw_attributes(event, results):
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
results.append(["When", str(when)])
results.append(["Deployment", event.deployment.name])
results.append(["Category", event.routing_key])
results.append(["Publisher", event.publisher])
results.append(["State", event.state])
results.append(["Event", event.event])
results.append(["Service", event.service])
results.append(["Host", event.host])
results.append(["UUID", event.instance])
results.append(["Req ID", event.request_id])
return results
def do_list_usage_deletes(request):
filter_args = {}
if "instance" in request.GET:
filter_args["instance"] = request.GET["instance"]
if len(filter_args) > 0:
deletes = models.InstanceDeletes.objects.filter(**filter_args)
else:
deletes = models.InstanceDeletes.objects.all()
results = []
results.append(["UUID", "Launched At", "Deleted At"])
for delete in deletes:
launched = None
if delete.launched_at:
launched = str(dt.dt_from_decimal(delete.launched_at))
deleted = None
if delete.deleted_at:
deleted = str(dt.dt_from_decimal(delete.deleted_at))
results.append([delete.instance, launched, deleted])
return rsp(results)
def append_generic_raw_attributes(event, results):
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
results.append(["When", str(when)])
results.append(["Deployment", event.deployment.name])
results.append(["Category", event.routing_key])
results.append(["Publisher", event.publisher])
results.append(["State", event.state])
results.append(["Event", event.event])
results.append(["Service", event.service])
results.append(["Host", event.host])
results.append(["UUID", event.instance])
results.append(["Req ID", event.request_id])
return results
def do_request(request):
request_id = request.GET['request_id']
if not utils.is_request_id_like(request_id):
msg = "%s is not request-id-like" % request_id
return error_response(400, 'Bad Request', msg)
events = models.RawData.objects.filter(request_id=request_id) \
.order_by('when')
results = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"]]
for e in events:
when = dt.dt_from_decimal(e.when)
results.append([e.id, routing_key_type(e.routing_key), str(when),
e.deployment.name, e.event, e.host, e.state,
e.old_state, e.old_task])
return rsp(json.dumps(results))
def do_uuid(request):
uuid = str(request.GET['uuid'])
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
related = models.RawData.objects.select_related().filter(instance=uuid)\
.order_by('when')
results = [["#", "?", "When", "Deployment", "Event", "Host", "State",
"State'", "Task'"]]
for e in related:
when = dt.dt_from_decimal(e.when)
results.append([e.id, routing_key_type(e.routing_key), str(when),
e.deployment.name, e.event, e.host, e.state,
e.old_state, e.old_task])
return rsp(json.dumps(results))
def do_request(request):
request_id = request.GET['request_id']
if not utils.is_request_id_like(request_id):
msg = "%s is not request-id-like" % request_id
return error_response(400, 'Bad Request', msg)
model = models.RawData.objects
filters = {'request_id': request_id}
_add_when_filters(request, filters)
events = model_search(request, model, filters, order_by='when')
results = [["#", "Instance", "When", "Event", "Host",
"Publisher"]]
for e in events:
when = dt.dt_from_decimal(e.when)
results.append([e.id, e.instance, str(when),
e.event, e.host, e.publisher])
return rsp(json.dumps(results))
def search(request):
service = str(request.GET.get('service', 'nova'))
field = request.GET.get('field')
value = request.GET.get('value')
model = _model_factory(service)
filters = {field: value}
_add_when_filters(request, filters)
results = []
try:
events = model_search(request, model, filters, order_by='-when')
for event in events:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
results = event.search_results(results, when, routing_key_status)
return rsp(json.dumps(results))
except ObjectDoesNotExist or FieldError:
return rsp([])
def do_uuid(request, service='nova'):
uuid = str(request.GET['uuid'])
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
model = _model_factory(service)
result = []
param = {}
if service == 'nova' or service == 'generic':
param = {'instance': uuid}
if service == 'glance':
param = {'uuid': uuid}
related = model.select_related().filter(**param).order_by('when')
for event in related:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
result = event.search_results(result, when, routing_key_status)
return rsp(json.dumps(result))
def search(request):
service = str(request.GET.get('service', 'nova'))
field = request.GET.get('field')
value = request.GET.get('value')
model = _model_factory(service)
filters = {field: value}
_add_when_filters(request, filters)
results = []
try:
events = model_search(request, model, filters, order_by='-when')
for event in events:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
results = event.search_results(results, when, routing_key_status)
return rsp(json.dumps(results))
except ObjectDoesNotExist:
return error_response(404, 'Not Found', ["The requested object does not exist"])
except FieldError:
return error_response(400, 'Bad Request', "The requested field '%s' does not exist for the corresponding object.\n"
"Note: The field names of database are case-sensitive." % field)
def search(request):
service = str(request.GET.get('service', 'nova'))
field = request.GET.get('field')
value = request.GET.get('value')
model = _model_factory(service)
filters = {field: value}
_add_when_filters(request, filters)
results = []
try:
events = model_search(request, model, filters, order_by='-when')
for event in events:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
results = event.search_results(results, when, routing_key_status)
return rsp(json.dumps(results))
except ObjectDoesNotExist:
return error_response(404, 'Not Found', ["The requested object does not exist"])
except FieldError:
return error_response(400, 'Bad Request', "The requested field '%s' does not exist for the corresponding object.\n"
"Note: The field names of database are case-sensitive." % field)
def do_list_usage_launches(request):
filter_args = {}
if "instance" in request.GET:
filter_args["instance"] = request.GET["instance"]
if len(filter_args) > 0:
launches = models.InstanceUsage.objects.filter(**filter_args)
else:
launches = models.InstanceUsage.objects.all()
results = []
results.append(["UUID", "Launched At", "Instance Type Id"])
for launch in launches:
launched = None
if launch.launched_at:
launched = str(dt.dt_from_decimal(launch.launched_at))
results.append([launch.instance, launched, launch.instance_type_id])
return rsp(results)
def search(request, service):
DEFAULT = 1000
field = request.GET.get('field')
value = request.GET.get('value')
limit = request.GET.get('limit', DEFAULT)
limit = int(limit)
model = _model_factory(service)
filter_para = {field: value}
results = []
try:
events = model.filter(**filter_para)
event_len = len(events)
if event_len > limit:
events = events[0:limit]
for event in events:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
results = event.search_results(results, when, routing_key_status)
return rsp(json.dumps(results))
except ObjectDoesNotExist or FieldError:
return rsp([])
def do_jsonreports_search(request):
try:
model = models.JsonReport
filters = _create_query_filters(request)
reports = model_search(request, model.objects, filters, order_by='-id')
results = [['Id', 'Start', 'End', 'Created', 'Name', 'Version']]
for report in reports:
results.append([
report.id,
datetime.datetime.strftime(report.period_start, UTC_FORMAT),
datetime.datetime.strftime(report.period_end, UTC_FORMAT),
datetime.datetime.strftime(dt.dt_from_decimal(report.created),
UTC_FORMAT), report.name,
report.version
])
except BadRequestException as be:
return error_response(400, 'Bad Request', be.message)
except ValidationError as ve:
return error_response(400, 'Bad Request', ve.messages[0])
return rsp(json.dumps(results))
def do_list_usage_launches(request):
filter_args = {}
if "instance" in request.GET:
uuid = request.GET["instance"]
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, "Bad Request", msg)
filter_args["instance"] = uuid
if len(filter_args) > 0:
launches = models.InstanceUsage.objects.filter(**filter_args)
else:
launches = models.InstanceUsage.objects.all()
results = [["UUID", "Launched At", "Instance Type Id"]]
for launch in launches:
launched = None
if launch.launched_at:
launched = str(dt.dt_from_decimal(launch.launched_at))
results.append([launch.instance, launched, launch.instance_type_id])
return rsp(results)
def do_uuid(request):
uuid = str(request.GET['uuid'])
service = str(request.GET.get('service', 'nova'))
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
model = _model_factory(service)
result = []
filters = {}
if service == 'nova' or service == 'generic':
filters = {'instance': uuid}
if service == 'glance':
filters = {'uuid': uuid}
_add_when_filters(request, filters)
related = model_search(request, model, filters,
related=True, order_by='when')
for event in related:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
result = event.search_results(result, when, routing_key_status)
return rsp(json.dumps(result))
def do_jsonreports_search(request):
try:
model = models.JsonReport
filters = _create_query_filters(request)
reports = model_search(request, model.objects, filters,
order_by='-id')
results = [['Id', 'Start', 'End', 'Created', 'Name', 'Version']]
for report in reports:
results.append([report.id,
datetime.datetime.strftime(
report.period_start, UTC_FORMAT),
datetime.datetime.strftime(
report.period_end, UTC_FORMAT),
datetime.datetime.strftime(
dt.dt_from_decimal(report.created),
UTC_FORMAT),
report.name,
report.version])
except BadRequestException as be:
return error_response(400, 'Bad Request', be.message)
except ValidationError as ve:
return error_response(400, 'Bad Request', ve.messages[0])
return rsp(json.dumps(results))
def _convert_model(model):
model_dict = model_to_dict(model)
for key in model_dict:
if isinstance(model_dict[key], decimal.Decimal):
model_dict[key] = str(dt.dt_from_decimal(model_dict[key]))
return model_dict
try:
con = mdb.connect(conf.STACKTACH_DB_HOST, conf.STACKTACH_DB_USERNAME,
conf.STACKTACH_DB_PASSWORD, conf.STACKTACH_DB_NAME)
with con:
cur = con.cursor(mdb.cursors.DictCursor)
sqlQuery = "Select tenant,service,json,`when` from stacktach_rawdata \
where `when` > " + str(utils.str_time_to_unix(str(LastRunTime))) + \
" and `when` <= " + str(utils.str_time_to_unix(str(ThisRunTime))) + \
" order by tenant,service,`when`;"
cur.execute(sqlQuery)
rows = cur.fetchall()
for row in rows:
# print row["tenant"], rename_service(row["service"]), datetime.date(dt.dt_from_decimal(row['when']))
exportDir = LogDir + row['tenant'] + '/' + rename_service(row['service'])
exportFile = exportDir + '/' + str(datetime.date(dt.dt_from_decimal(row['when']))) + '.log'
if not os.path.exists(exportDir):
os.makedirs(exportDir)
with open(exportFile, 'a+') as outfile:
outfile.write(row['json'])
with open(LastRunFile, 'w+') as outfile:
outfile.write(str(ThisRunTime))
except mdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit(1)
# zip older files
files = [y for x in os.walk(LogDir) for y in glob(os.path.join(x[0], '*.log'))]
findstr = str(datetime.date(ThisRunTime)) + '.log'
for file in files:
if file.find(findstr) == -1: