def ready(self):
if not TEST:
from CouncilTag.ingest.models import Agenda, Committee
from CouncilTag.api.utils import getLocationBasedDate
from celery.schedules import crontab
from CouncilTag.celery import schedule_process_pdf, app
print("SETTING UP CELERY ASYNC TASKS!")
app.conf.beat_schedule = {}
app.conf.timezone = 'UTC'
committees = Committee.objects.all()
for committee in committees:
agendas = Agenda.objects.filter(committee=committee,
processed=False)
for agenda in agendas:
agenda.processed = True
agenda.save()
dt = getLocationBasedDate(agenda.meeting_time,
committee.cutoff_offset_days,
committee.cutoff_hour,
committee.cutoff_minute,
committee.location_tz)
schedule_process_pdf.apply_async(
(committee.name, agenda.meeting_id), eta=dt)
app.conf.beat_schedule[committee.name] = {
'task': 'CouncilTag.celery.schedule_committee_processing',
'schedule': crontab(hour='*/2', minute='35'),
'args': (committee.name, )
}
def ready(self):
if DEBUG:
from CouncilTag.ingest.models import Agenda, Committee
from CouncilTag.api.utils import getLocationBasedDate
from celery.schedules import crontab
from CouncilTag.celery import schedule_process_pdf, app
from celery.task.control import inspect
log.error("SETTING UP CELERY ASYNC TASKS!")
i = inspect()
log.error(i.scheduled())
app.conf.beat_schedule = {}
app.conf.timezone = 'UTC'
committees = Committee.objects.all()
for committee in committees:
agendas = Agenda.objects.filter(committee=committee,
processed=False)
for agenda in agendas:
dt = getLocationBasedDate(agenda.meeting_time,
committee.cutoff_offset_days,
committee.cutoff_hour,
committee.cutoff_minute,
committee.location_tz)
dt = dt + timedelta(minutes=5)
log.error(
f"scheduling pdf processing for: {dt} for: {committee.name}"
)
dt_utc = datetime.fromtimestamp(dt.timestamp(),
tz=pytz.timezone('UTC'))
exists = r.get(f"{committee.name}-{agenda.meeting_time}")
if exists is None:
r.set(f"{committee.name}-{agenda.meeting_time}",
True,
ex=3 * 60)
schedule_process_pdf.apply_async(
(committee.name, agenda.meeting_id), eta=dt_utc)
log.error(f"scheduled pdf processing")
else:
log.error(
f'{committee.name} {agenda.meeting_id} already queued for pdf'
)
app.conf.beat_schedule[committee.name] = {
'task': 'CouncilTag.celery.schedule_committee_processing',
'schedule': crontab(hour='*', minute='*/30'),
'args': (committee.name, )
}
log.error(i.scheduled())
def handle(self, *args, **options):
now = datetime.now()
# Find agenda between today and next 10 days.
now_time_stamp = int(now.timestamp())
committees = Committee.objects.all()
if len(committees) == 0:
print(f"No committees to check agendas for")
return
for committee in committees:
print(committee.name)
cutoff_offset_days = committee.cutoff_offset_days
cutoff_hour = committee.cutoff_hour
cutoff_minute = committee.cutoff_minute
location_tz = committee.location_tz
time_delta = 1200 # 1200 seconds, 20m * 60s/m # 21600 is 6 hours
upcoming_agendas = Agenda.objects.filter(processed=False)
if (len(upcoming_agendas) == 0):
# No agendas
return
for agenda in upcoming_agendas:
meeting_time = agenda.meeting_time
cutoff_datetime = getLocationBasedDate(meeting_time,
cutoff_offset_days,
cutoff_hour,
cutoff_minute,
location_tz)
cutoff_timestamp = int(cutoff_datetime.timestamp())
if (now_time_stamp - time_delta < cutoff_timestamp <
now_time_stamp + time_delta):
agenda.processed = True
agenda.save()
upcoming_agenda_items = AgendaItem.objects.filter(
agenda=agenda)
if (len(upcoming_agenda_items) == 0):
print(
"No upcoming agenda items on {meeting_time} for {committee.name}"
)
continue
writePdfForAgendaItems(upcoming_agenda_items, committee,
agenda)
def processAgendasForYears(years, committee_name):
try:
committee = Committee.objects.get(name=committee_name)
except ObjectDoesNotExist:
committee = Committee(name="Santa Monica City Council", email="*****@*****.**",
cutoff_offset_days=0, cutoff_hour=11, cutoff_minute=59, location_lat=34.024212, location_lng=-118.496475, location_tz="America/Los_Angeles")
committee.save()
for year in years:
agenda_values = get_data(year=year)
if agenda_values is None:
print(f"No agendas/items for {year}")
continue
for time, agenda in agenda_values.items():
if len(agenda) > 0:
found_agenda = Agenda.objects.filter(
meeting_time=time).first()
print(found_agenda)
if found_agenda is None:
found_agenda = Agenda(meeting_time=time)
found_agenda.committee = committee
found_agenda.meeting_id = agenda[0]['MeetingID']
dt = getLocationBasedDate(found_agenda.meeting_time, committee.cutoff_offset_days,
committee.cutoff_hour, committee.cutoff_minute, committee.location_tz)
dt = dt + timedelta(minutes=5)
log.error(f"scheduling pdf processing for: {dt} for: {committee.name}")
dt_utc = datetime.fromtimestamp(dt.timestamp(), tz=pytz.timezone('UTC'))
exists = r.get(f"{committee.name}-{found_agenda.meeting_time}")
found_agenda.save()
if exists is None:
r.set(f"{committee.name}-{found_agenda.meeting_time}", True, ex=3*60)
schedule_process_pdf.apply_async(
(committee.name, found_agenda.meeting_id), eta=dt_utc)
log.error(f"scheduled pdf processing")
else:
log.error(f'{committee.name} {found_agenda.meeting_id} already queued for pdf in utils')
for ag_item in agenda:
save_agendaitem(ag_item, found_agenda, time)
found_agenda.save()