def handle(self, *args, **options): if os.access('/tmp/extract_readings.lock', os.R_OK): t = os.path.getmtime('/tmp/extract_readings.lock') created = datetime.datetime.fromtimestamp(t) if (datetime.datetime.now() - created).total_seconds() > 4 * 60 * 60: print('extract_readings: Stale lock - removing...') os.remove('/tmp/extract_readings.lock') else: return touch('/tmp/extract_readings.lock') tag = 'extracted_readings' start = timezone.now() payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains='ingest_error')[:500]) end = timezone.now() query_time = (end - start).total_seconds() while len(payloads) > 0: touch('/tmp/extract_readings.lock') start = timezone.now() for payload in payloads: payload.ingest_readings() end = timezone.now() perf_values = {} perf_values['num_extracted'] = len(payloads) perf_values['query_time'] = query_time perf_values['extraction_time'] = (end - start).total_seconds() append_performance_sample('system', 'reading_ingestion_performance', end, perf_values) start = timezone.now() payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains='ingest_error')[:500]) end = timezone.now() query_time = (end - start).total_seconds() readings = PurpleRobotReading.objects.filter(guid=None)[:250] for reading in readings: reading.update_guid() readings = PurpleRobotReading.objects.filter(guid=None)[:250] os.remove('/tmp/extract_readings.lock')
def log_reading(reading): yesterday = timezone.now() - datetime.timedelta(days=1) if reading.logged > yesterday: append_performance_sample(reading.user_id, reading.probe, reading.logged, { 'sample_count': 1 })
def handle(self, *args, **options): if os.access('/tmp/gather_statistics.lock', os.R_OK): t = os.path.getmtime('/tmp/gather_statistics.lock') created = datetime.datetime.fromtimestamp(t) if (datetime.datetime.now() - created).total_seconds() > 6 * 60 * 60: print('gather_statistics: Stale lock - removing...') os.remove('/tmp/gather_statistics.lock') else: return touch('/tmp/gather_statistics.lock') tz = pytz.timezone(settings.TIME_ZONE) now = arrow.get(timezone.now().astimezone(tz)) start_today = now.replace(hour=0, minute=0, second=0, microsecond=0).datetime start_hour = now.datetime - datetime.timedelta(hours=1) yesterday = now.datetime - datetime.timedelta(days=1) count = PurpleRobotPayload.objects.filter(added__gte=start_hour).count() append_performance_sample('system', 'uploads_hour', timezone.now(), { 'count': count }) count = PurpleRobotPayload.objects.filter(added__gte=start_today).count() append_performance_sample('system', 'uploads_today', timezone.now(), { 'count': count }) tag = 'extracted_into_database' skip_tag = 'extracted_into_database_skip' count = PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() append_performance_sample('system', 'pending_mirror_payloads', timezone.now(), { 'count': count }) count = PurpleRobotPayload.objects.filter(process_tags__contains=skip_tag).count() append_performance_sample('system', 'skipped_mirror_payloads', timezone.now(), { 'count': count }) week = now.datetime - datetime.timedelta(days=7) day = now.datetime - datetime.timedelta(days=1) half_day = now.datetime - datetime.timedelta(hours=12) quarter_day = now.datetime - datetime.timedelta(hours=6) hour = now.datetime - datetime.timedelta(hours=1) week_count = PurpleRobotPayload.objects.filter(added__lt=week).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() day_count = PurpleRobotPayload.objects.filter(added__gte=week, added__lt=day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() half_day_count = PurpleRobotPayload.objects.filter(added__gte=day, added__lt=half_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() quarter_day_count = PurpleRobotPayload.objects.filter(added__gte=half_day, added__lt=quarter_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() hour_count = PurpleRobotPayload.objects.filter(added__gte=quarter_day, added__lt=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() less_hour_count = PurpleRobotPayload.objects.filter(added__gte=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() append_performance_sample('system', 'pending_mirror_ages', timezone.now(), { 'week_count': week_count, 'day_count': day_count, 'half_day_count': half_day_count, 'quarter_day_count': quarter_day_count, 'hour_count': hour_count, 'less_hour_count': less_hour_count }) tag = 'extracted_readings' skip_tag = 'ingest_error' count = PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() append_performance_sample('system', 'pending_ingest_payloads', timezone.now(), { 'count': count }) count = PurpleRobotPayload.objects.filter(process_tags__contains=skip_tag).count() append_performance_sample('system', 'skipped_ingest_payloads', timezone.now(), { 'count': count }) week_count = PurpleRobotPayload.objects.filter(added__lt=week).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() day_count = PurpleRobotPayload.objects.filter(added__gte=week, added__lt=day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() half_day_count = PurpleRobotPayload.objects.filter(added__gte=day, added__lt=half_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() quarter_day_count = PurpleRobotPayload.objects.filter(added__gte=half_day, added__lt=quarter_day).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() hour_count = PurpleRobotPayload.objects.filter(added__gte=quarter_day, added__lt=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() less_hour_count = PurpleRobotPayload.objects.filter(added__gte=hour).exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).count() append_performance_sample('system', 'pending_ingest_ages', timezone.now(), { 'week_count': week_count, 'day_count': day_count, 'half_day_count': half_day_count, 'quarter_day_count': quarter_day_count, 'hour_count': hour_count, 'less_hour_count': less_hour_count }) uptime = os.popen("/usr/bin/uptime").read() uptime = uptime.split('load average: ')[1].split(" ") load_minute = float(uptime[0].replace(',', '').strip()) load_five = float(uptime[1].replace(',', '').strip()) load_fifteen = float(uptime[2].replace(',', '').strip()) append_performance_sample('system', 'server_performance', timezone.now(), { 'load_minute': load_minute, 'load_five': load_five, 'load_fifteen': load_fifteen }) counts = [] index_time = start_today while (index_time + datetime.timedelta(seconds=(15 * 60))) < timezone.now(): end = index_time + datetime.timedelta(seconds=(30 * 60)) plot = index_time + datetime.timedelta(seconds=(15 * 60)) counts.append({ 'date': plot.isoformat(), 'count': PurpleRobotPayload.objects.filter(added__gte=index_time, added__lt=end).count() }) index_time = end append_performance_sample('system', 'payload_uploads', timezone.now(), { 'counts': counts }) os.remove('/tmp/gather_statistics.lock')
def handle(self, *args, **options): try: settings.PURPLE_ROBOT_FLAT_MIRROR except AttributeError: return if os.access('/tmp/extract_into_database.lock', os.R_OK): timestamp = os.path.getmtime('/tmp/extract_into_database.lock') created = datetime.datetime.fromtimestamp(timestamp) if (datetime.datetime.now() - created).total_seconds() > 60 * 60 * 8: print 'extract_into_database: Stale lock - removing...' os.remove('/tmp/extract_into_database.lock') else: return touch('/tmp/extract_into_database.lock') tag = 'extracted_into_database' skip_tag = 'extracted_into_database_skip' start = timezone.now() payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).order_by('-added')[:250]) end = timezone.now() query_time = (end - start).total_seconds() extractor_times = {} extractor_counts = {} local_db = 0.0 remote_db = 0.0 local_app = 0.0 while len(payloads) > 0: touch('/tmp/extract_into_database.lock') extractor_times = {} extractor_counts = {} local_db = query_time remote_db = 0.0 local_app = 0.0 start = timezone.now() for payload in payloads: cpu_start = datetime.datetime.now() items = json.loads(payload.payload) has_all_extractors = True missing_extractors = [] for item in items: if 'PROBE' in item and 'GUID' in item: probe_name = my_slugify(item['PROBE']).replace('edu_northwestern_cbits_purple_robot_manager_probes_', '') found = False if (probe_name in EXTRACTORS) is True: if EXTRACTORS[probe_name] is not None: found = True else: found = False else: EXTRACTORS[probe_name] = None for app in settings.INSTALLED_APPS: try: probe = importlib.import_module(app + '.management.commands.extractors.' + probe_name) EXTRACTORS[probe_name] = probe found = True except ImportError: pass if found is False: has_all_extractors = False if (probe_name in missing_extractors) == False: missing_extractors.append(probe_name) else: has_all_extractors = False missing_extractors.append('Unknown Probe') if has_all_extractors: for item in items: if 'PROBE' in item and 'GUID' in item: probe_name = my_slugify(item['PROBE']).replace('edu_northwestern_cbits_purple_robot_manager_probes_', '') probe = EXTRACTORS[probe_name] write_start = datetime.datetime.now() local_app += (write_start - cpu_start).total_seconds() exists = True if settings.PURPLE_ROBOT_DISABLE_DATA_CHECKS: exists = False else: exists = probe.exists(settings.PURPLE_ROBOT_FLAT_MIRROR, payload.user_id, item) cpu_start = datetime.datetime.now() remote_db += (cpu_start - write_start).total_seconds() if EXTRACTORS[probe_name] is not None and exists is False: write_start = datetime.datetime.now() local_app += (write_start - cpu_start).total_seconds() EXTRACTORS[probe_name].insert(settings.PURPLE_ROBOT_FLAT_MIRROR, payload.user_id, item, check_exists=(settings.PURPLE_ROBOT_DISABLE_DATA_CHECKS is False)) cpu_start = datetime.datetime.now() remote_db += (cpu_start - write_start).total_seconds() duration = 0.0 if probe_name in extractor_times: duration = extractor_times[probe_name] duration += (cpu_start - write_start).total_seconds() extractor_times[probe_name] = duration count = 0.0 if probe_name in extractor_counts: count = extractor_counts[probe_name] count += 1 extractor_counts[probe_name] = count tags = payload.process_tags if tags is None or tags.find(tag) == -1: if tags is None or len(tags) == 0: tags = tag else: tags += ' ' + tag payload.process_tags = tags read_start = datetime.datetime.now() local_app += (read_start - cpu_start).total_seconds() payload.save() cpu_start = datetime.datetime.now() local_db += (cpu_start - read_start).total_seconds() else: tags = payload.process_tags if tags is None or tags.find(skip_tag) == -1: if tags is None or len(tags) == 0: tags = skip_tag else: tags += ' ' + skip_tag payload.process_tags = tags read_start = datetime.datetime.now() local_app += (read_start - cpu_start).total_seconds() payload.save() cpu_start = datetime.datetime.now() local_db += (cpu_start - read_start).total_seconds() if len(missing_extractors) > 0: print 'MISSING EXTRACTORS: ' + str(missing_extractors) end = timezone.now() perf_values = {} perf_values['num_mirrored'] = len(payloads) perf_values['query_time'] = query_time perf_values['local_db'] = local_db perf_values['remote_db'] = remote_db perf_values['local_app'] = local_app perf_values['extraction_time'] = (end - start).total_seconds() append_performance_sample('system', 'reading_mirror_performance', end, perf_values) start = timezone.now() payloads = list(PurpleRobotPayload.objects.exclude(process_tags__contains=tag).exclude(process_tags__contains=skip_tag).order_by('-added')[:250]) end = timezone.now() query_time = (end - start).total_seconds() os.remove('/tmp/extract_into_database.lock')