def run_batch(self, batch): """runs the single batch prints batch title then loops through the query set and prints progress in % afterwards there will be a short summary """ sys.stdout.write(batch['title'].encode('utf-8')) changed_count = 0 checked_count = 0 total_count = batch['query_set'].count() if total_count == 0: transaction.commit() return for item in batch['query_set'].all(): item_changed = batch['function'](item) transaction.commit() if item_changed: changed_count += 1 checked_count += 1 progress = 100 * float(checked_count) / float(total_count) console.print_progress(FORMAT_STRING, progress) print FORMAT_STRING % 100 if changed_count: print batch['changed_count_message'] % changed_count else: print batch['nothing_changed_message']
def run_batch(self, batch): """runs the single batch prints batch title then loops through the query set and prints progress in % afterwards there will be a short summary """ sys.stdout.write(batch['title'].encode('utf-8')) changed_count = 0 checked_count = 0 total_count = batch['query_set'].count() if total_count == 0: transaction.commit() return for item in batch['query_set'].all(): item_changed = batch['function'](item) transaction.commit() if item_changed: changed_count += 1 checked_count += 1 progress = 100*float(checked_count)/float(total_count) console.print_progress(FORMAT_STRING, progress) print FORMAT_STRING % 100 if changed_count: print batch['changed_count_message'] % changed_count else: print batch['nothing_changed_message']
def run_batch(cls, batch): """runs the single batch prints batch title then loops through the query set and prints progress in % afterwards there will be a short summary """ sys.stdout.write(batch['title'].encode('utf-8')) changed_count = 0 checked_count = 0 total_count = batch['query_set'].count() if total_count == 0: return for item in batch['query_set'].all(): with transaction.atomic(): # pylint: disable=no-member item_changed = batch['function'](item) if item_changed: changed_count += 1 checked_count += 1 console.print_progress(checked_count, total_count) console.print_progress(checked_count, total_count) if changed_count: print(batch['changed_count_message'] % changed_count) else: print(batch['nothing_changed_message'])
def handle_noargs(self, **options): tags = models.Tag.objects.all() count = 0 print "Searching for unused tags:", total = tags.count() deleted_tags = list() for tag in tags: if not tag.threads.exists(): deleted_tags.append(tag.name) tag.delete() transaction.commit() count += 1 progress = 100*float(count)/float(total) console.print_progress('%6.2f%%', progress) print '%6.2f%%' % 100 if deleted_tags: found_count = len(deleted_tags) if found_count == 1: print "Found an unused tag %s" % deleted_tags[0] else: sys.stdout.write("Found %d unused tags" % found_count) if found_count > 50: print ", first 50 are:", print ', '.join(deleted_tags[:50]) + '.' else: print ": " + ', '.join(deleted_tags) + '.' print "Deleted." else: print "Did not find any."
def run_command(self): """method that runs the actual command""" #go through tags and find character case duplicates and eliminate them tagnames = models.Tag.objects.values_list('name', flat = True) for name in tagnames: dupes = models.Tag.objects.filter(name__iexact = name) first_tag = dupes[0] if dupes.count() > 1: line = 'Found duplicate tags for %s:' % first_tag.name for idx in xrange(1, dupes.count()): print dupes[idx].name + ' ', dupes[idx].delete() print '' if askbot_settings.FORCE_LOWERCASE_TAGS: lowercased_name = first_tag.name.lower() if first_tag.name != lowercased_name: print 'Converting tag %s to lower case' % first_tag.name first_tag.name = lowercased_name first_tag.save() transaction.commit() #go through questions and fix tag records on each questions = models.Question.objects.all() checked_count = 0 found_count = 0 total_count = questions.count() print "Searching for questions with inconsistent tag records:", for question in questions: tags = question.tags.all() denorm_tag_set = set(question.get_tag_names()) norm_tag_set = set(question.tags.values_list('name', flat=True)) if norm_tag_set != denorm_tag_set: if question.last_edited_by: user = question.last_edited_by timestamp = question.last_edited_at else: user = question.author timestamp = question.added_at tagnames = forms.TagNamesField().clean(question.tagnames) question.update_tags( tagnames = tagnames, user = user, timestamp = timestamp ) question.tagnames = tagnames question.save() found_count += 1 transaction.commit() checked_count += 1 progress = 100*float(checked_count)/float(total_count) console.print_progress(FORMAT_STRING, progress) print FORMAT_STRING % 100 if found_count: print '%d problem questions found, tag records restored' % found_count else: print 'Did not find any problems'
def handle_noargs(self, **options): '''deletes old sessions''' quiet = options.get('quiet', False) expired_session_count = Session.objects.filter(expire_date__lt=datetime.now()).count() expired_session_list= Session.objects.filter(expire_date__lt=datetime.now()).values_list('session_key', flat=True) transaction.commit() if not quiet: print "There are %d expired sessions" % expired_session_count range_limit = len(expired_session_list) - 1 higher_limit = lower_limit = 0 for i in range(DELETE_LIMIT, range_limit, DELETE_LIMIT): lower_limit = i higher_limit = lower_limit + DELETE_LIMIT sublist = expired_session_list[lower_limit:higher_limit] Session.objects.filter(session_key__in = sublist).delete() transaction.commit() if not quiet: print_progress(higher_limit-1, expired_session_count) if higher_limit < expired_session_list: sublist = expired_session_list[higher_limit:expired_session_count] Session.objects.filter(session_key__in = sublist).delete() #print_progress(expired_session_count, expired_session_count) transaction.commit() if not quiet: print "sessions cleared"
def handle_noargs(self, **options): tags = models.Tag.objects.all() count = 0 print "Searching for unused tags:", total = tags.count() deleted_tags = list() for tag in tags: if tag.questions.all().count() == 0: deleted_tags.append(tag.name) tag.delete() transaction.commit() count += 1 progress = 100 * float(count) / float(total) console.print_progress('%6.2f%%', progress) print '%6.2f%%' % 100 if deleted_tags: found_count = len(deleted_tags) if found_count == 1: print "Found an unused tag %s" % deleted_tags[0] else: sys.stdout.write("Found %d unused tags" % found_count) if found_count > 50: print ", first 50 are:", print ', '.join(deleted_tags[:50]) + '.' else: print ": " + ', '.join(deleted_tags) + '.' print "Deleted." else: print "Did not find any."
def run_batch(self, batch): """runs the single batch prints batch title then loops through the query set and prints progress in % afterwards there will be a short summary """ sys.stdout.write(batch["title"].encode("utf-8")) changed_count = 0 checked_count = 0 total_count = batch["query_set"].count() if total_count == 0: transaction.commit() return for item in batch["query_set"].all(): item_changed = batch["function"](item) transaction.commit() if item_changed: changed_count += 1 checked_count += 1 progress = 100 * float(checked_count) / float(total_count) console.print_progress(checked_count, total_count) console.print_progress(checked_count, total_count) if changed_count: print batch["changed_count_message"] % changed_count else: print batch["nothing_changed_message"]
def run_command(self): """method that runs the actual command""" #go through tags and find character case duplicates and eliminate them tagnames = models.Tag.objects.values_list('name', flat=True) for name in tagnames: dupes = models.Tag.objects.filter(name__iexact=name) first_tag = dupes[0] if dupes.count() > 1: line = 'Found duplicate tags for %s: ' % first_tag.name print line, for idx in xrange(1, dupes.count()): print dupes[idx].name + ' ', dupes[idx].delete() print '' if askbot_settings.FORCE_LOWERCASE_TAGS: lowercased_name = first_tag.name.lower() if first_tag.name != lowercased_name: print 'Converting tag %s to lower case' % first_tag.name first_tag.name = lowercased_name first_tag.save() transaction.commit() #go through exercises and fix tag records on each threads = models.Thread.objects.all() checked_count = 0 found_count = 0 total_count = threads.count() print "Searching for exercises with inconsistent tag records:", for thread in threads: tags = thread.tags.all() denorm_tag_set = set(thread.get_tag_names()) norm_tag_set = set(thread.tags.values_list('name', flat=True)) if norm_tag_set != denorm_tag_set: if thread.last_edited_by: user = thread.last_edited_by timestamp = thread.last_edited_at else: user = thread.author timestamp = thread.added_at tagnames = forms.TagNamesField().clean(thread.tagnames) thread.update_tags(tagnames=tagnames, user=user, timestamp=timestamp) thread.tagnames = tagnames thread.save() found_count += 1 transaction.commit() checked_count += 1 progress = 100 * float(checked_count) / float(total_count) console.print_progress(FORMAT_STRING, progress) print FORMAT_STRING % 100 if found_count: print '%d problem exercises found, tag records restored' % found_count else: print 'Did not find any problems'
def run_command(self, lang): """method that runs the actual command""" #go through tags and find character case duplicates and eliminate them translation.activate(lang) tagnames = models.Tag.objects.filter( language_code=lang ).values_list('name', flat=True) self.admin = get_admin() #1) first we go through all tags and #either fix or delete illegal tags found_count = 0 for name in tagnames: try: tag = models.Tag.objects.get( name=name, language_code=lang ) except models.Tag.DoesNotExist: #tag with this name was already deleted, #because it was an invalid duplicate version #of other valid tag continue fixed_name = get_valid_tag_name(tag) #if fixed name is empty after cleaning, delete the tag if fixed_name == '': print 'Deleting invalid tag: %s' % name tag.delete() found_count += 1 continue if fixed_name != name: print 'Renaming tag: %s -> %s' % (name, fixed_name) #if tag name changed, see if there is a duplicate #with the same name, in which case we re-assign questions #with the current tag to that other duplicate #then delete the current tag as no longer used if fixed_name != name: try: duplicate_tag = models.Tag.objects.get( name=fixed_name, language_code=lang ) except models.Tag.DoesNotExist: pass self.retag_threads([tag], duplicate_tag) tag.delete() found_count += 1 continue #if there are case variant dupes, we assign questions #from the case variants to the current tag and #delete the case variant tags dupes = models.Tag.objects.filter( name__iexact=fixed_name, language_code=lang ).exclude(pk=tag.id) dupes_count = dupes.count() if dupes_count: self.retag_threads(dupes, tag) dupes.delete() found_count += dupes_count if tag.name != fixed_name: tag.name = fixed_name tag.save() transaction.commit() #2) go through questions and fix tag records on each # and recalculate all the denormalised tag names on threads threads = models.Thread.objects.all() checked_count = 0 total_count = threads.count() print "Searching for questions with inconsistent copies of tag records:", for thread in threads: #make sure that denormalized tag set is the same as normalized #we just add both the tags together and try to apply them #to the question tags = thread.tags.all() denorm_tag_set = set(thread.get_tag_names()) norm_tag_set = set(thread.tags.values_list('name', flat=True)) if norm_tag_set != denorm_tag_set: denorm_tag_set.update(norm_tag_set) cleaned_tag_set = set( models.Tag.objects.filter( name__in=denorm_tag_set, language_code=lang ).values_list('name', flat=True) ) self.admin.retag_question( question=thread._question_post(), tags=' '.join(cleaned_tag_set) ) transaction.commit() checked_count += 1 console.print_progress(checked_count, total_count) console.print_progress(checked_count, total_count) if found_count: print '%d problem questions found, tag records restored' % found_count else: print 'Did not find any problems'
def run_command(self, lang): """method that runs the actual command""" #go through tags and find character case duplicates and eliminate them translation.activate(lang) tagnames = models.Tag.objects.filter(language_code=lang).values_list( 'name', flat=True) self.admin = get_admin() #1) first we go through all tags and #either fix or delete illegal tags found_count = 0 for name in tagnames: try: tag = models.Tag.objects.get(name=name, language_code=lang) except models.Tag.DoesNotExist: #tag with this name was already deleted, #because it was an invalid duplicate version #of other valid tag continue fixed_name = get_valid_tag_name(tag) #if fixed name is empty after cleaning, delete the tag if fixed_name == '': print 'Deleting invalid tag: %s' % name tag.delete() found_count += 1 continue if fixed_name != name: print 'Renaming tag: %s -> %s' % (name, fixed_name) #if tag name changed, see if there is a duplicate #with the same name, in which case we re-assign questions #with the current tag to that other duplicate #then delete the current tag as no longer used if fixed_name != name: try: duplicate_tag = models.Tag.objects.get(name=fixed_name, language_code=lang) except models.Tag.DoesNotExist: pass self.retag_threads([tag], duplicate_tag) tag.delete() found_count += 1 continue #if there are case variant dupes, we assign questions #from the case variants to the current tag and #delete the case variant tags dupes = models.Tag.objects.filter( name__iexact=fixed_name, language_code=lang).exclude(pk=tag.id) dupes_count = dupes.count() if dupes_count: self.retag_threads(dupes, tag) dupes.delete() found_count += dupes_count if tag.name != fixed_name: tag.name = fixed_name tag.save() transaction.commit() #2) go through questions and fix tag records on each # and recalculate all the denormalised tag names on threads threads = models.Thread.objects.all() checked_count = 0 total_count = threads.count() print "Searching for questions with inconsistent copies of tag records:", for thread in threads: #make sure that denormalized tag set is the same as normalized #we just add both the tags together and try to apply them #to the question tags = thread.tags.all() denorm_tag_set = set(thread.get_tag_names()) norm_tag_set = set(thread.tags.values_list('name', flat=True)) if norm_tag_set != denorm_tag_set: denorm_tag_set.update(norm_tag_set) cleaned_tag_set = set( models.Tag.objects.filter(name__in=denorm_tag_set, language_code=lang).values_list( 'name', flat=True)) self.admin.retag_question(question=thread._question_post(), tags=' '.join(cleaned_tag_set)) transaction.commit() checked_count += 1 console.print_progress(checked_count, total_count) console.print_progress(checked_count, total_count) if found_count: print '%d problem questions found, tag records restored' % found_count else: print 'Did not find any problems'