def handle(self, *args, **options):
if os.access('/tmp/extract_readings.lock', os.R_OK):
t = os.path.getmtime('/tmp/extract_readings.lock')
created = datetime.datetime.fromtimestamp(t)
if (datetime.datetime.now() - created).total_seconds() > 4 * 60 * 60:
print('extract_readings: Stale lock - removing...')
os.remove('/tmp/extract_readings.lock')
else:
return
touch('/tmp/extract_readings.lock')
tag = 'extracted_readings'
start = timezone.now()
payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains='ingest_error')[:500])
end = timezone.now()
query_time = (end - start).total_seconds()
while len(payloads) > 0:
touch('/tmp/extract_readings.lock')
start = timezone.now()
for payload in payloads:
payload.ingest_readings()
end = timezone.now()
perf_values = {}
perf_values['num_extracted'] = len(payloads)
perf_values['query_time'] = query_time
perf_values['extraction_time'] = (end - start).total_seconds()
append_performance_sample('system', 'reading_ingestion_performance', end, perf_values)
start = timezone.now()
payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains='ingest_error')[:500])
end = timezone.now()
query_time = (end - start).total_seconds()
readings = PurpleRobotReading.objects.filter(guid=None)[:250]
for reading in readings:
reading.update_guid()
readings = PurpleRobotReading.objects.filter(guid=None)[:250]
os.remove('/tmp/extract_readings.lock')
def log_reading(reading):
yesterday = timezone.now() - datetime.timedelta(days=1)
if reading.logged > yesterday:
append_performance_sample(reading.user_id, reading.probe, reading.logged, { 'sample_count': 1 })
def handle(self, *args, **options):
if os.access('/tmp/gather_statistics.lock', os.R_OK):
t = os.path.getmtime('/tmp/gather_statistics.lock')
created = datetime.datetime.fromtimestamp(t)
if (datetime.datetime.now() - created).total_seconds() > 6 * 60 * 60:
print('gather_statistics: Stale lock - removing...')
os.remove('/tmp/gather_statistics.lock')
else:
return
touch('/tmp/gather_statistics.lock')
tz = pytz.timezone(settings.TIME_ZONE)
now = arrow.get(timezone.now().astimezone(tz))
start_today = now.replace(hour=0, minute=0, second=0, microsecond=0).datetime
start_hour = now.datetime - datetime.timedelta(hours=1)
yesterday = now.datetime - datetime.timedelta(days=1)
count = PurpleRobotPayload.objects.filter(added__gte=start_hour).count()
append_performance_sample('system', 'uploads_hour', timezone.now(), { 'count': count })
count = PurpleRobotPayload.objects.filter(added__gte=start_today).count()
append_performance_sample('system', 'uploads_today', timezone.now(), { 'count': count })
tag = 'extracted_into_database'
skip_tag = 'extracted_into_database_skip'
count = PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'pending_mirror_payloads', timezone.now(), { 'count': count })
count = PurpleRobotPayload.objects.filter(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'skipped_mirror_payloads', timezone.now(), { 'count': count })
week = now.datetime - datetime.timedelta(days=7)
day = now.datetime - datetime.timedelta(days=1)
half_day = now.datetime - datetime.timedelta(hours=12)
quarter_day = now.datetime - datetime.timedelta(hours=6)
hour = now.datetime - datetime.timedelta(hours=1)
week_count = PurpleRobotPayload.objects.filter(added__lt=week).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
day_count = PurpleRobotPayload.objects.filter(added__gte=week, added__lt=day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
half_day_count = PurpleRobotPayload.objects.filter(added__gte=day, added__lt=half_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
quarter_day_count = PurpleRobotPayload.objects.filter(added__gte=half_day, added__lt=quarter_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
hour_count = PurpleRobotPayload.objects.filter(added__gte=quarter_day, added__lt=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
less_hour_count = PurpleRobotPayload.objects.filter(added__gte=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'pending_mirror_ages', timezone.now(), {
'week_count': week_count,
'day_count': day_count,
'half_day_count': half_day_count,
'quarter_day_count': quarter_day_count,
'hour_count': hour_count,
'less_hour_count': less_hour_count })
tag = 'extracted_readings'
skip_tag = 'ingest_error'
count = PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'pending_ingest_payloads', timezone.now(), { 'count': count })
count = PurpleRobotPayload.objects.filter(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'skipped_ingest_payloads', timezone.now(), { 'count': count })
week_count = PurpleRobotPayload.objects.filter(added__lt=week).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
day_count = PurpleRobotPayload.objects.filter(added__gte=week, added__lt=day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
half_day_count = PurpleRobotPayload.objects.filter(added__gte=day, added__lt=half_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
quarter_day_count = PurpleRobotPayload.objects.filter(added__gte=half_day, added__lt=quarter_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
hour_count = PurpleRobotPayload.objects.filter(added__gte=quarter_day, added__lt=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
less_hour_count = PurpleRobotPayload.objects.filter(added__gte=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count()
append_performance_sample('system', 'pending_ingest_ages', timezone.now(), {
'week_count': week_count,
'day_count': day_count,
'half_day_count': half_day_count,
'quarter_day_count': quarter_day_count,
'hour_count': hour_count,
'less_hour_count': less_hour_count })
uptime = os.popen("/usr/bin/uptime").read()
uptime = uptime.split('load average: ')[1].split(" ")
load_minute = float(uptime[0].replace(',', '').strip())
load_five = float(uptime[1].replace(',', '').strip())
load_fifteen = float(uptime[2].replace(',', '').strip())
append_performance_sample('system', 'server_performance', timezone.now(), { 'load_minute': load_minute, 'load_five': load_five, 'load_fifteen': load_fifteen })
counts = []
index_time = start_today
while (index_time + datetime.timedelta(seconds=(15 * 60))) < timezone.now():
end = index_time + datetime.timedelta(seconds=(30 * 60))
plot = index_time + datetime.timedelta(seconds=(15 * 60))
counts.append({ 'date': plot.isoformat(), 'count': PurpleRobotPayload.objects.filter(added__gte=index_time, added__lt=end).count() })
index_time = end
append_performance_sample('system', 'payload_uploads', timezone.now(), { 'counts': counts })
os.remove('/tmp/gather_statistics.lock')
def handle(self, *args, **options):
try:
settings.PURPLE_ROBOT_FLAT_MIRROR
except AttributeError:
return
if os.access('/tmp/extract_into_database.lock', os.R_OK):
timestamp = os.path.getmtime('/tmp/extract_into_database.lock')
created = datetime.datetime.fromtimestamp(timestamp)
if (datetime.datetime.now() - created).total_seconds() > 60 * 60 * 8:
print 'extract_into_database: Stale lock - removing...'
os.remove('/tmp/extract_into_database.lock')
else:
return
touch('/tmp/extract_into_database.lock')
tag = 'extracted_into_database'
skip_tag = 'extracted_into_database_skip'
start = timezone.now()
payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).order_by('-added')[:250])
end = timezone.now()
query_time = (end - start).total_seconds()
extractor_times = {}
extractor_counts = {}
local_db = 0.0
remote_db = 0.0
local_app = 0.0
while len(payloads) > 0:
touch('/tmp/extract_into_database.lock')
extractor_times = {}
extractor_counts = {}
local_db = query_time
remote_db = 0.0
local_app = 0.0
start = timezone.now()
for payload in payloads:
cpu_start = datetime.datetime.now()
items = json.loads(payload.payload)
has_all_extractors = True
missing_extractors = []
for item in items:
if 'PROBE' in item and 'GUID' in item:
probe_name = my_slugify(item['PROBE']).replace('edu_northwestern_cbits_purple_robot_manager_probes_', '')
found = False
if (probe_name in EXTRACTORS) is True:
if EXTRACTORS[probe_name] is not None:
found = True
else:
found = False
else:
EXTRACTORS[probe_name] = None
for app in settings.INSTALLED_APPS:
try:
probe = importlib.import_module(app + '.management.commands.extractors.' + probe_name)
EXTRACTORS[probe_name] = probe
found = True
except ImportError:
pass
if found is False:
has_all_extractors = False
if (probe_name in missing_extractors) == False:
missing_extractors.append(probe_name)
else:
has_all_extractors = False
missing_extractors.append('Unknown Probe')
if has_all_extractors:
for item in items:
if 'PROBE' in item and 'GUID' in item:
probe_name = my_slugify(item['PROBE']).replace('edu_northwestern_cbits_purple_robot_manager_probes_', '')
probe = EXTRACTORS[probe_name]
write_start = datetime.datetime.now()
local_app += (write_start - cpu_start).total_seconds()
exists = True
if settings.PURPLE_ROBOT_DISABLE_DATA_CHECKS:
exists = False
else:
exists = probe.exists(settings.PURPLE_ROBOT_FLAT_MIRROR, payload.user_id, item)
cpu_start = datetime.datetime.now()
remote_db += (cpu_start - write_start).total_seconds()
if EXTRACTORS[probe_name] is not None and exists is False:
write_start = datetime.datetime.now()
local_app += (write_start - cpu_start).total_seconds()
EXTRACTORS[probe_name].insert(settings.PURPLE_ROBOT_FLAT_MIRROR, payload.user_id, item, check_exists=(settings.PURPLE_ROBOT_DISABLE_DATA_CHECKS is False))
cpu_start = datetime.datetime.now()
remote_db += (cpu_start - write_start).total_seconds()
duration = 0.0
if probe_name in extractor_times:
duration = extractor_times[probe_name]
duration += (cpu_start - write_start).total_seconds()
extractor_times[probe_name] = duration
count = 0.0
if probe_name in extractor_counts:
count = extractor_counts[probe_name]
count += 1
extractor_counts[probe_name] = count
tags = payload.process_tags
if tags is None or tags.find(tag) == -1:
if tags is None or len(tags) == 0:
tags = tag
else:
tags += ' ' + tag
payload.process_tags = tags
read_start = datetime.datetime.now()
local_app += (read_start - cpu_start).total_seconds()
payload.save()
cpu_start = datetime.datetime.now()
local_db += (cpu_start - read_start).total_seconds()
else:
tags = payload.process_tags
if tags is None or tags.find(skip_tag) == -1:
if tags is None or len(tags) == 0:
tags = skip_tag
else:
tags += ' ' + skip_tag
payload.process_tags = tags
read_start = datetime.datetime.now()
local_app += (read_start - cpu_start).total_seconds()
payload.save()
cpu_start = datetime.datetime.now()
local_db += (cpu_start - read_start).total_seconds()
if len(missing_extractors) > 0:
print 'MISSING EXTRACTORS: ' + str(missing_extractors)
end = timezone.now()
perf_values = {}
perf_values['num_mirrored'] = len(payloads)
perf_values['query_time'] = query_time
perf_values['local_db'] = local_db
perf_values['remote_db'] = remote_db
perf_values['local_app'] = local_app
perf_values['extraction_time'] = (end - start).total_seconds()
append_performance_sample('system', 'reading_mirror_performance', end, perf_values)
start = timezone.now()
payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).order_by('-added')[:250])
end = timezone.now()
query_time = (end - start).total_seconds()
os.remove('/tmp/extract_into_database.lock')
