def test_couch_synclogs(self): synclog = SyncLog(domain='test', user_id='user1', date=datetime.datetime(2015, 7, 1, 0, 0)) SyncLog.get_db().save_doc(synclog) self.assertEqual(self._sql_count(), 0) self.assertEqual(self._couch_count(), 1) delete_synclog(synclog._id) self.assertEqual(self._sql_count(), 0) self.assertEqual(self._couch_count(), 0) with self.assertRaises(MissingSyncLog): delete_synclog(synclog._id)
def get_all_sync_logs_docs(): assert settings.UNIT_TESTING all_sync_log_ids = [row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", reduce=False, )] return iter_docs(SyncLog.get_db(), all_sync_log_ids)
def rows(self): base_link_url = '{}?q={{id}}'.format(reverse('global_quick_find')) user_id = self.request.GET.get('individual') if not user_id: return [] # security check get_document_or_404(CommCareUser, self.domain, user_id) sync_log_ids = [row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id], descending=True, reduce=False, limit=10 )] def _sync_log_to_row(sync_log): def _fmt_duration(duration): if isinstance(duration, int): return format_datatables_data( '<span class="{cls}">{text}</span>'.format( cls=_bootstrap_class(duration or 0, 60, 20), text=_('{} seconds').format(duration), ), duration ) else: return format_datatables_data( '<span class="label">{text}</span>'.format( text=_("Unknown"), ), -1, ) def _fmt_id(sync_log_id): href = base_link_url.format(id=sync_log_id) return '<a href="{href}" target="_blank">{id:.5}...</a>'.format( href=href, id=sync_log_id ) num_cases = len(sync_log.cases_on_phone) columns = [ _fmt_date(sync_log.date), format_datatables_data(num_cases, num_cases), _fmt_duration(sync_log.duration), ] if self.show_extra_columns: columns.append(_fmt_id(sync_log.get_id)) return columns return [ _sync_log_to_row(SyncLog.wrap(sync_log_json)) for sync_log_json in iter_docs(SyncLog.get_db(), sync_log_ids) ]
def get_all_sync_logs_docs(): all_sync_log_ids = [ row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", reduce=False, ) ] return iter_docs(SyncLog.get_db(), all_sync_log_ids)
def delete_sync_logs(before_date, limit=1000): from casexml.apps.phone.dbaccessors.sync_logs_by_user import get_synclog_ids_before_date from casexml.apps.phone.models import SyncLog from dimagi.utils.couch.database import iter_bulk_delete_with_doc_type_verification sync_log_ids = get_synclog_ids_before_date(before_date, limit) return iter_bulk_delete_with_doc_type_verification(SyncLog.get_db(), sync_log_ids, 'SyncLog', chunksize=5)
def delete_sync_logs(before_date, limit=1000, num_tries=10): from casexml.apps.phone.dbaccessors.sync_logs_by_user import get_synclog_ids_before_date from casexml.apps.phone.models import SyncLog from dimagi.utils.couch.database import iter_bulk_delete_with_doc_type_verification for i in range(num_tries): try: sync_log_ids = get_synclog_ids_before_date(before_date, limit) return iter_bulk_delete_with_doc_type_verification( SyncLog.get_db(), sync_log_ids, 'SyncLog', chunksize=25) except BulkSaveError: pass raise CouldNotPruneSyncLogs()
def get_user_sync_history_pillow(pillow_id='UpdateUserSyncHistoryPillow', **kwargs): """ This gets a pillow which iterates through all synclogs """ couch_db = SyncLog.get_db() change_feed = CouchChangeFeed(couch_db, include_docs=True) checkpoint = PillowCheckpoint('synclog', change_feed.sequence_format) form_processor = UserSyncHistoryProcessor() return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=form_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100), )
def rows(self): user_id = self.request.GET.get('individual') if not user_id: return [] # security check get_document_or_404(CommCareUser, self.domain, user_id) sync_log_ids = [row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id], descending=True, reduce=False, )] def _sync_log_to_row(sync_log): def _fmt_duration(duration): if isinstance(duration, int): return format_datatables_data( '<span class="{cls}">{text}</span>'.format( cls=_bootstrap_class(duration or 0, 20, 60), text=_('{} seconds').format(duration), ), duration ) else: return format_datatables_data( '<span class="label">{text}</span>'.format( text=_("Unknown"), ), -1, ) num_cases = len(sync_log.cases_on_phone) return [ _fmt_date(sync_log.date), format_datatables_data(num_cases, num_cases), _fmt_duration(sync_log.duration), ] return [ _sync_log_to_row(SyncLog.wrap(sync_log_json)) for sync_log_json in iter_docs(SyncLog.get_db(), sync_log_ids) ]
def handle(self, filename, *args, **kwargs): database = SyncLog.get_db() all_sync_log_ids = [ row['id'] for row in database.view( 'phone/sync_logs_by_user', reduce=False, include_docs=False) ] total_count = len(all_sync_log_ids) headers = [ 'date', 'user', 'cases', 'dependent cases', 'total cases', 'initial', 'duration', 'duration per case (ms/case)', ] with open(filename, 'wb') as f: writer = csv.writer(f, dialect=csv.excel) writer.writerow(headers) for i, sync_log_dict in enumerate( iter_docs(database, all_sync_log_ids, 500)): duration = sync_log_dict.get('duration') cases = len(sync_log_dict.get('cases_on_phone', [])) dependent_cases = len( sync_log_dict.get('dependent_cases_on_phone', [])) total_cases = cases + dependent_cases if duration and total_cases: average_time = float(duration) * 1000 / float(total_cases) writer.writerow([ (sync_log_dict.get('date') or '1980-01-01')[:10], # strip times off of the dates sync_log_dict.get('user_id'), cases, dependent_cases, cases + dependent_cases, bool(sync_log_dict.get('previous_log_id')), duration, '{0:.2f}'.format(average_time) ]) if i % 500 == 0: print 'processed {}/{} logs'.format(i, total_count)
def handle(self, filename, *args, **kwargs): # Doesn't work since this queries from Couch # Todo: Migrate to SQL raise CommandError("This doesn't work since the synclogs are now migrated to SQL") database = SyncLog.get_db() all_sync_log_ids = [ row['id'] for row in database.view('phone/sync_logs_by_user', reduce=False, include_docs=False) ] total_count = len(all_sync_log_ids) headers = [ 'date', 'user', 'cases', 'dependent cases', 'total cases', 'initial', 'duration', 'duration per case (ms/case)', ] with open(filename, 'wb') as f: writer = csv.writer(f, dialect=csv.excel) writer.writerow( headers ) for i, sync_log_dict in enumerate(iter_docs(database, all_sync_log_ids, 500)): duration = sync_log_dict.get('duration') cases = len(sync_log_dict.get('cases_on_phone', [])) dependent_cases = len(sync_log_dict.get('dependent_cases_on_phone', [])) total_cases = cases + dependent_cases if duration and total_cases: average_time = float(duration) * 1000 / float(total_cases) writer.writerow([ (sync_log_dict.get('date') or '1980-01-01')[:10], # strip times off of the dates sync_log_dict.get('user_id'), cases, dependent_cases, cases + dependent_cases, bool(sync_log_dict.get('previous_log_id')), duration, '{0:.2f}'.format(average_time) ]) if i % 500 == 0: print('processed {}/{} logs'.format(i, total_count))
def delete_all_sync_logs(): # handle with care _delete_all(SyncLog.get_db(), 'phone/sync_logs_by_user')
def delete_all_sync_logs(cls): logger.debug("Deleting all synclogs") cls._delete_all_from_view(SyncLog.get_db(), 'phone/sync_logs_by_user')
def delete_all_sync_logs(cls): cls._delete_all(SyncLog.get_db(), 'phone/sync_logs_by_user')
def delete_all_sync_logs(cls): logger.debug("Deleting all synclogs") cls._delete_all(SyncLog.get_db(), 'phone/sync_logs_by_user')
def record_iter(cls, start_datetime, end_datetime): synclog_ids = get_synclog_ids_by_date(start_datetime, end_datetime) return iter_docs(SyncLog.get_db(), synclog_ids)
def delete_all_sync_logs(): _delete_all(SyncLog.get_db(), 'phone/sync_logs_by_user')
def synclog_view(view_name, **params): return combine_views([SyncLog.get_db(), get_db(None)], view_name, **params)
def delete_all_sync_logs(cls): SyncLogSQL.objects.all().delete() cls._delete_all_from_view(SyncLog.get_db(), 'phone/sync_logs_by_user')
def rows(self): base_link_url = '{}?q={{id}}'.format(reverse('global_quick_find')) user_id = self.request.GET.get('individual') if not user_id: return [] # security check get_document_or_404(CommCareUser, self.domain, user_id) sync_log_ids = [ row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id], descending=True, reduce=False, limit=self.limit, ) ] def _sync_log_to_row(sync_log): def _fmt_duration(duration): if isinstance(duration, int): return format_datatables_data( '<span class="{cls}">{text}</span>'.format( cls=_bootstrap_class(duration or 0, 60, 20), text=_('{} seconds').format(duration), ), duration) else: return format_datatables_data( '<span class="label">{text}</span>'.format( text=_("Unknown"), ), -1, ) def _fmt_id(sync_log_id): href = base_link_url.format(id=sync_log_id) return '<a href="{href}" target="_blank">{id:.5}...</a>'.format( href=href, id=sync_log_id) def _fmt_error_info(sync_log): if not sync_log.had_state_error: return u'<span class="label label-success">✓</span>' else: return ( u'<span class="label label-important">X</span>' u'State error {}<br>Expected hash: {:.10}...').format( _naturaltime_with_hover(sync_log.error_date), sync_log.error_hash, ) num_cases = sync_log.case_count() columns = [ _fmt_date(sync_log.date), format_datatables_data(num_cases, num_cases), _fmt_duration(sync_log.duration), ] if self.show_extra_columns: columns.append(_fmt_id(sync_log.get_id)) columns.append(sync_log.log_format) columns.append( _fmt_id(sync_log.previous_log_id) if sync_log. previous_log_id else '---') columns.append(_fmt_error_info(sync_log)) columns.append('{:.10}...'.format(sync_log.get_state_hash())) columns.append(_naturaltime_with_hover( sync_log.last_submitted)) columns.append(u'{}<br>{:.10}'.format( _naturaltime_with_hover(sync_log.last_cached), sync_log.hash_at_last_cached)) return columns return [ _sync_log_to_row(properly_wrap_sync_log(sync_log_json)) for sync_log_json in iter_docs(SyncLog.get_db(), sync_log_ids) ]
def rows(self): base_link_url = '{}?q={{id}}'.format(reverse('global_quick_find')) user_id = self.request.GET.get('individual') if not user_id: return [] # security check get_document_or_404(CommCareUser, self.domain, user_id) sync_log_ids = [row['id'] for row in SyncLog.view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id], descending=True, reduce=False, limit=self.limit, )] def _sync_log_to_row(sync_log): def _fmt_duration(duration): if isinstance(duration, int): return format_datatables_data( '<span class="{cls}">{text}</span>'.format( cls=_bootstrap_class(duration or 0, 60, 20), text=_('{} seconds').format(duration), ), duration ) else: return format_datatables_data( '<span class="label">{text}</span>'.format( text=_("Unknown"), ), -1, ) def _fmt_id(sync_log_id): href = base_link_url.format(id=sync_log_id) return '<a href="{href}" target="_blank">{id:.5}...</a>'.format( href=href, id=sync_log_id ) def _fmt_error_info(sync_log): if not sync_log.had_state_error: return u'<span class="label label-success">✓</span>' else: return (u'<span class="label label-important">X</span>' u'State error {}<br>Expected hash: {:.10}...').format( _naturaltime_with_hover(sync_log.error_date), sync_log.error_hash, ) num_cases = sync_log.case_count() columns = [ _fmt_date(sync_log.date), format_datatables_data(num_cases, num_cases), _fmt_duration(sync_log.duration), ] if self.show_extra_columns: columns.append(_fmt_id(sync_log.get_id)) columns.append(sync_log.log_format) columns.append(_fmt_id(sync_log.previous_log_id) if sync_log.previous_log_id else '---') columns.append(_fmt_error_info(sync_log)) columns.append('{:.10}...'.format(sync_log.get_state_hash())) columns.append(_naturaltime_with_hover(sync_log.last_submitted)) columns.append(u'{}<br>{:.10}'.format(_naturaltime_with_hover(sync_log.last_cached), sync_log.hash_at_last_cached)) return columns return [ _sync_log_to_row(properly_wrap_sync_log(sync_log_json)) for sync_log_json in iter_docs(SyncLog.get_db(), sync_log_ids) ]