def handle(self, *args, **options):
# monday = '{} {} 1'.format(ISO_CALENDAR[0], ISO_CALENDAR[1])
# query = Task.objects.filter(
# starting__gte=datetime.fromtimestamp(
# time.mktime(time.strptime(monday, '%Y %W %w'))))
query = Task.objects.filter(starting__gte=TODAY)
data = [('state', 'starting', 'end', 'dataset_id', 'dataset_name',
'resource_id', 'resource_name', 'error')]
for item in query:
resource = Resource.objects.get(id=item.extras['resource'])
dataset = resource.dataset
data.append(
(item.state, item.starting.isoformat(), item.end.isoformat(),
dataset.id, dataset.title, resource.id, resource.title,
item.extras.get('error', None)))
f = StringIO()
csv.writer(f).writerows(data)
mail_instance = Mail.objects.get(template_name='email_task')
mail = EmailMessage(mail_instance.subject, mail_instance.message,
FROM_EMAIL, get_admins_mails())
mail.attach('log.csv', f.getvalue(), 'text/csv')
mail.send()
def check_resources_last_update(*args, **kwargs):
delta_map = {
'5mn': relativedelta(minutes=5),
'15mn': relativedelta(minutes=15),
'20mn': relativedelta(minutes=20),
'30mn': relativedelta(minutes=30),
'1hour': relativedelta(hours=1),
'3hours': relativedelta(hours=3),
'6hours': relativedelta(hours=6),
'daily': relativedelta(days=1),
'weekly': relativedelta(days=7),
'bimonthly': relativedelta(days=15),
'monthly': relativedelta(months=1),
'quarterly': relativedelta(months=3),
'biannual': relativedelta(months=6),
'annual': relativedelta(year=1),
}
data = []
for resource in Resource.objects.filter(**kwargs):
delta = delta_map.get(resource.sync_frequency)
if not delta:
continue
delay = timezone.now() - (resource.last_update + delta)
if delay.total_seconds() > 0:
row = (
resource.dataset.slug,
resource.ckan_id,
resource.sync_frequency,
resource.last_update,
delay,
)
data.append(row)
if not data:
return
# else:
col = (
'dataset_slug',
'resource_uuid',
'sync_frequency',
'last_update',
'delay',
)
data.insert(0, col)
f = StringIO()
csv.writer(f).writerows(data)
mail_instance = Mail.objects.get(
template_name='resources_update_with_delay')
mail = EmailMessage(mail_instance.subject, mail_instance.message,
settings.DEFAULT_FROM_EMAIL, get_admins_mails())
mail.attach('log.csv', f.getvalue(), 'text/csv')
mail.send()
def send_sync_report_mail(*args, **kwargs):
"""Envoyer un e-mail contenant les dernières Tasks exécutées ;
par exemple suite un une synchronisation des ressources exécutée
avec le script `sync_resources`.
"""
tasks_tracking = TaskTracking.objects.filter(
task='celeriac.tasks.save_resource',
start__gte=timezone.datetime.today().date(),
end__isnull=False,
)
csv_data = [('state',
'starting',
'end',
'dataset_id',
'dataset_name',
'resource_id',
'resource_name',
'error')]
for task_tracking in tasks_tracking:
try:
resource = Resource.objects.get(pk=task_tracking.detail['kwargs']['pk'])
except KeyError as e:
logger.error("Malformed JSON: please check instance TaskTracking '%d'" % task_tracking.pk)
logger.warning("Error was ignored.")
continue
except Resource.DoesNotExist as e:
logger.exception(e)
logger.warning("Error was ignored.")
continue
# else:
csv_data.append((
task_tracking.state,
task_tracking.start.isoformat(),
task_tracking.end.isoformat(),
resource.dataset.id,
resource.dataset.title,
resource.id,
resource.title,
task_tracking.detail.get('error', None),
))
f = StringIO()
csv.writer(f).writerows(csv_data)
mail_instance = Mail.objects.get(template_name='email_task')
mail = EmailMessage(
mail_instance.subject,
mail_instance.message,
DEFAULT_FROM_EMAIL,
get_admins_mails())
mail.attach('log.csv', f.getvalue(), 'text/csv')
if ENABLE_SENDING_MAIL:
try:
mail.send()
except SMTPDataError as e:
logger.error(e)
# Activer l'exception lorsque gérée par l'application.
# return MailError()
else:
logger.warning("Sending mail is disable.")
def check_resources_last_update(*args, **kwargs):
csv_data = [('dataset_slug',
'resource_uuid',
'sync_frequency',
'last_update',
'delay')]
for resource in Resource.objects.filter(**kwargs):
delta = {
# '5mn': relativedelta(minutes=5),
# '15mn': relativedelta(minutes=15),
# '20mn': relativedelta(minutes=20),
# '30mn': relativedelta(minutes=30),
'1hour': relativedelta(hours=1),
'3hours': relativedelta(hours=3),
'6hours': relativedelta(hours=6),
'daily': relativedelta(days=1),
'weekly': relativedelta(days=7),
'bimonthly': relativedelta(days=15),
'monthly': relativedelta(months=1),
'quarterly': relativedelta(months=3),
'biannual': relativedelta(months=6),
'annual': relativedelta(year=1),
}.get(resource.sync_frequency)
if not delta:
continue
delay = timezone.now() - (resource.last_update + delta)
if delay.total_seconds() > 0:
csv_data.append((
resource.dataset.slug,
resource.ckan_id,
resource.sync_frequency,
resource.last_update,
delay,
))
if not csv_data: # Nothing to do
return
f = StringIO()
csv.writer(f).writerows(csv_data)
mail_instance = Mail.objects.get(template_name='resources_update_with_delay')
mail = EmailMessage(
mail_instance.subject,
mail_instance.message,
DEFAULT_FROM_EMAIL,
get_admins_mails())
mail.attach('log.csv', f.getvalue(), 'text/csv')
if ENABLE_SENDING_MAIL:
try:
mail.send()
except SMTPDataError as e:
logger.error(e)
# Activer l'exception lorsque gérée par l'application.
# return MailError()
else:
logger.warning("Sending mail is disable.")