def handle(self, *args, **options):
dryrun = options['dryrun']
q = Post.objects.all_articleless()
# Keep retrying until we get a legitimate error code
# explaining the failure.
q = q.filter(article_content_error_code__isnull=True)
#q = q.filter(article_content_success__isnull=True)
#TODO:retry article_content_success=False but with error_code__isnull=False?
year = options['year']
month = options['month']
if year:
q = q.filter(date_published__year=year)
if month:
q = q.filter(date_published__month=month)
#q = q.only('id', )
q = q.order_by('-date_published')
total = q.count()
i = 0
success_count = 0 # successfully retrieved non-empty content
error_count = 0 # any type of exception was thrown
meh_count = 0 # no errors were thrown, even if we didn't get content
print '%i posts without an article.' % (total,)
if dryrun:
return
for post in q.iterator():
i += 1
print '\rProcessing post %i (%i of %i, %i success, %i errors, %i mehs)...' \
% (post.id, i, total, success_count, error_count, meh_count),
sys.stdout.flush()
Job.update_progress(total_parts=total, total_parts_complete=i)
try:
post.retrieve_article_content(force=options['force'])
success_count += bool(len((post.article_content or '').strip()))
meh_count += not bool(len((post.article_content or '').strip()))
except urllib2.HTTPError, e:
error_count += 1
print>>sys.stderr
print>>sys.stderr, 'Error: Unable to retrieve %s: %s' % (post.link, e)
post.article_content_error_code = e.code
post.article_content_error_reason = e.reason
post.article_content_success = False
post.save()
except Exception, e:
post.article_content_success = False
ferr = StringIO()
traceback.print_exc(file=ferr)
post.article_content = None
post.article_content_error = ferr.getvalue()
post.save()
error_count += 1
print>>sys.stderr
print>>sys.stderr, 'Error: Unable to retrieve %s: %s' % (post.link, e)
def print_progress(self, clear=True, newline=True):
if self.last_progress_refresh \
and (datetime.now() - self.last_progress_refresh).seconds < 0.5:
return
bar_length = 10
if clear:
sys.stdout.write('\033[2J\033[H') #clear screen
sys.stdout.write('Importing attributes\n')
for stripe, msg_parts in sorted(self.progress.items()):
(current, total, sub_current, sub_total, eta, message) = msg_parts
sub_status = ''
if total:
if not eta:
start_time = self.start_times[stripe]
current_seconds = time.time() - start_time
total_seconds = float(total)/current*current_seconds
remaining_seconds = int(total_seconds - current_seconds)
eta = timezone.now() + timedelta(seconds=remaining_seconds)
self.stripe_counts[stripe] = (current, total)
percent = current/float(total)
bar = ('=' * int(percent * bar_length)).ljust(bar_length)
percent = int(percent * 100)
else:
eta = eta or '?'
percent = 0
bar = ('=' * int(percent * bar_length)).ljust(bar_length)
percent = '?'
total = '?'
if sub_current and sub_total:
sub_status = '(subtask %s of %s) ' % (sub_current, sub_total)
sys.stdout.write(
(('' if newline else '\r') + \
"%s [%s] %s of %s %s%s%% eta=%s: %s"+('\n' if newline else '')) \
% (stripe, bar, current, total, sub_status, percent, eta, message))
sys.stdout.flush()
self.last_progress_refresh = datetime.now()
# Update job.
overall_current_count = 0
overall_total_count = 0
for stripe, (current, total) in six.iteritems(self.stripe_counts):
overall_current_count += current
overall_total_count += total
#print('overall_current_count:',overall_current_count
#print('overall_total_count:',overall_total_count
if overall_total_count and Job:
Job.update_progress(
total_parts_complete=overall_current_count,
total_parts=overall_total_count,
)
def testTimezone(self):
self.assertEqual(settings.USE_TZ, True)
j = Job()
j.command = "test_sleeper"
j.frequency = "MINUTELY"
j.enabled = True
j.params = "interval:10"
j.next_run = datetime.datetime(2014, 6, 27, 14, 31, 4)
j.save()
# Test someone turning-on timezone awareness after job was created.
try:
settings.USE_TZ = False
j = Job()
j.command = "test_sleeper"
j.frequency = "MINUTELY"
j.enabled = True
j.save()
self.assertTrue(j.next_run)
settings.USE_TZ = True
j.params = "interval:10"
j.next_run = datetime.datetime(2014, 6, 27, 14, 31, 4)#, tzinfo=timezone.get_default_timezone())
j.save()
finally:
settings.USE_TZ = True
def handle(self, *args, **options):
seconds = int(options['seconds'])
for i in range(seconds):
Job.update_progress(total_parts=seconds, total_parts_complete=i)
print('%i of %i' % (i, seconds))
time.sleep(1)
def handle(self, *args, **options):
seconds = int(options['seconds'])
for i in xrange(seconds):
Job.update_progress(total_parts=seconds, total_parts_complete=i)
print('%i of %i' % (i, seconds))
time.sleep(1)
def testTimezone(self):
self.assertEqual(settings.USE_TZ, True)
j = Job()
j.command = "test_sleeper"
j.frequency = "MINUTELY"
j.enabled = True
j.params = "interval:10"
j.next_run = datetime(2014, 6, 27, 14, 31, 4)
j.save()
# Test someone turning-on timezone awareness after job was created.
try:
settings.USE_TZ = False
j = Job()
j.command = "test_sleeper"
j.frequency = "MINUTELY"
j.enabled = True
j.save()
self.assertTrue(j.next_run)
settings.USE_TZ = True
j.params = "interval:10"
j.next_run = datetime(2014, 6, 27, 14, 31, 4)
j.save()
finally:
settings.USE_TZ = True
