class UnLoadJob(CronJobBase):
schedule = Schedule(run_every_mins=1)
code = 'warehouse.unload_job'
def do(self):
need_to_loaded_cargo: QuerySet[
CargoLoadTimeSheet] = CargoLoadTimeSheet.objects.get(
date_load__lt=now(), completed=False).all()
for cargo_sheet in need_to_loaded_cargo:
ware_point: WarehousePoint = WarehousePoint.objects.filter(
type='VOID').first()
cargo: Cargo = Cargo.objects.filter(
factory_io_id=cargo_sheet.factory_io_id).first()
answ = cargo_unload({
"point": ware_point.point_num,
"shelving": ware_point.shelving
})
if answ.get('error'):
cargo_sheet.completed = True
cargo.place = ware_point
cargo_sheet.save()
cargo.save()
class MorningAnnouncement(CronJobBase):
schedule = Schedule(run_at_times=['08:30'])
code = 'crossbot.morning_announcement'
def format_message(self, announce_data):
msgs = ['Good morning crossworders!']
msgs += [
'{u} is currently on a {n}-day win streak! {emoji}'.format(
u=u, n=len(streak), emoji=':fire:' * len(streak))
for u, streak in announce_data['streaks']
]
# now add the other winners
also = ' also' if announce_data['streaks'] else ''
if announce_data['winners_today']:
is_are = 'are' if len(announce_data['winners_today']) > 1 else 'is'
msgs.append('{} {}{} winning'.format(
comma_and(announce_data['winners_today']), is_are, also))
if announce_data['winners_yesterday']:
msgs.append(
comma_and(announce_data['winners_yesterday']) + also +
' won yesterday.')
msgs.append("Think you can beat them? Play today's:")
for game in announce_data['links']:
msgs.append("{} : {}".format(game, announce_data['links'][game]))
return '\n'.join(msgs)
def do(self):
now = timezone.localtime()
announce_data = MiniCrosswordTime.announcement_data(now)
message = self.format_message(announce_data)
# TODO dont hardcode
channel = 'C58PXJTNU'
response = post_message(channel, {'text': message})
return "Ran morning announcement at {}\n{}".format(now, message)
class UpdateDelegates(CronJobBase):
RUN_EVERY_MINS = 30
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'ark_delegate_manager.update_delegates'
def do(self):
'''
update delegate statistics and save in DB
'''
try:
api.use('ark')
ark_node.set_connection(host=config.CONNECTION['HOST'],
database=config.CONNECTION['DATABASE'],
user=config.CONNECTION['USER'],
password=config.CONNECTION['PASSWORD'])
ark_node.set_delegate(
address=config.DELEGATE['ADDRESS'],
pubkey=config.DELEGATE['PUBKEY'],
)
all_delegates = utils.api_call(
api.Delegate.getDelegates)['delegates']
for delegate in all_delegates:
delegate_obj = ark_delegate_manager.models.ArkDelegates.objects.get_or_create(
pubkey=delegate['publicKey'])[0]
delegate_obj.username = delegate['username']
delegate_obj.address = delegate['address']
delegate_obj.ark_votes = delegate['vote']
delegate_obj.producedblocks = delegate['producedblocks']
delegate_obj.missedblocks = delegate['missedblocks']
delegate_obj.productivity = delegate['productivity']
delegate_obj.rank = delegate['rate']
delegate_obj.voters = len(
ark_node.Delegate.voters(delegate['address'])) + 1
delegate_obj.save()
except Exception:
logger.exception('Error during UpdateDelegates')
class RefreshContacts(CronJobBase):
""" Refresh contacts. """
RUN_EVERY_MINS = 10
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'app.crons.RefreshContacts'
def do(self):
print('Refreshing contacts...')
sf_client = SalesforceClient()
dorg_client = DiscoverOrgClient()
accounts = Account.objects.all()
for account in accounts:
sf_contacts = sf_client.get_contacts(account)
dorg_contacts = dorg_client.search(account)
if sf_contacts and dorg_contacts:
contacts = sf_contacts + dorg_contacts
elif sf_contacts and not dorg_contacts:
contacts = sf_contacts
elif dorg_contacts and not sf_contacts:
contacts = dorg_contacts
else:
contacts = []
for contact in contacts:
obj, created = Contact.objects.get_or_create(
account=account, name=contact['name'])
if created:
contact['status'] = 'enrich'
else:
contact['status'] = obj.status
Contact.objects.filter(pk=obj.pk).update(**contact)
class Enrich(CronJobBase):
""" Enriches contacts in all projects. """
RUN_EVERY_MINS = 10
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'api.jobs.Enrich'
def do(self):
print('Enriching projects...')
do = DiscoverOrgClient()
projects = Project.objects.filter(status='running bots')
for project in projects:
sheet, worksheet = get_session(project.url, project.worksheet)
contacts = worksheet.get_all_records()
for contact in contacts:
if contact['Status'] != 'Completed':
contact = do.enrich(contact)
if type(contact) is str:
continue
else:
continue
for k, v in contact.items():
row = contacts.index(contact) + 2
col = worksheet.find(k).col
worksheet.update_cell(row, col, v)
time.sleep(1)
time.sleep(3)
project.status = 'in progress'
project.save()
class GetContacts(CronJobBase):
""" Gets contacts for accounts from Salesforce. """
RUN_EVERY_MINS = 10
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = "api.jobs.GetContacts"
def do(self):
sf = SalesforceClient()
do = DiscoverOrgClient()
accounts = Account.objects.exclude(prep=None)
bar = Bar("CONTACT COLLECTION", max=accounts.count())
for account in accounts:
sf_contacts = sf.get_contacts(account)
if str(account.updated) != str(datetime.today().strftime("%Y-%m-%d")):
do_contacts = do.get_contacts(account)
account.updated = datetime.today().strftime("%Y-%m-%d")
account.save()
else:
do_contacts = []
contacts = sf_contacts + do_contacts
for contact in contacts:
obj, created = Contact.objects.get_or_create(
account=account, name=contact["name"]
)
contact["status"] = contact["status"] if created else obj.status
contact["ctype"] = contact["ctype"] if created else obj.ctype
Contact.objects.filter(pk=obj.pk).update(**contact)
bar.next()
bar.finish()
class SendPaymentRequest(SentryCronJobBase):
schedule = Schedule(run_at_times=[settings.CRON_SEND_PAYMENT_REQUEST_TIME])
code = 'leprikon.cronjobs.SendPaymentRequest'
def dojob(self):
today = date.today()
for registration in chain(
set(
registration_period.registration
for registration_period in CourseRegistrationPeriod.objects.filter(
registration__approved__isnull=False,
registration__canceled__isnull=True,
payment_requested=False,
period__due_from__lte=today,
).select_related('registration')
),
EventRegistration.objects.filter(
approved__isnull=False,
canceled__isnull=True,
payment_requested__isnull=True,
subject__event__due_from__lte=today,
),
(
registration
for registration in OrderableRegistration.objects.filter(
approved__isnull=False,
canceled__isnull=True,
payment_requested__isnull=True,
).select_related('subject__orderable')
if registration.get_due_from() <= today
),
):
try:
registration.request_payment(None)
except Exception:
print_exc()
capture_exception()
class UploadPriviledgeCronJob(CronJobBase):
"""
This class grants user upload file privilege if he checks the
upload checkbox in sign up form
"""
RUN_EVERY_MINS = 1440 # run every day
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'bhagirath.translation.update_priviledge_cron_job' # a unique code
def job(self):
check = UserProfile.objects.filter(contributor = 1)
uncheck = UserProfile.objects.filter(contributor = 0)
i = UserProfile.objects.filter(contributor=1).count()
k = UserProfile.objects.filter(contributor = 0).count()
j = 0
while j < i:
c = check[j]
usr = User.objects.get(username=c.user)
perm_id = Permission.objects.get(codename = 'add_task')
if usr.has_perm('translation.add_task'):
pass
else:
usr.user_permissions.add(perm_id)
usr.save()
j += 1
j = 0
while j < k:
u = uncheck[j]
usr = User.objects.get(username=u.user)
if not usr.has_perm('translation.add_task'):
pass
else:
usr.user_permissions.remove(perm_id)
usr.save()
j += 1
class Sync(CronJobBase):
""" Sync data with Salesforce. """
RUN_EVERY_MINS = 10
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'app.crons.Sync'
def do(self):
print('Syncing database...')
sf_client = SalesforceClient()
accounts = Account.objects.filter(status='queued')
for account in accounts:
contacts = Contact.objects.filter(account=account, status='queued')
for contact in contacts:
sf_client.create_contact(account, contact)
contact.status = 'done'
contact.save()
sf_client.complete_account(account)
account.status = 'done'
account.save()
class SampleCronJob(CronJobBase):
RUN_EVERY_MINS = 1440 # every 24 hours
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'insurance.sampleCron'
def create_dues(self):
for policy in Policy.objects.all():
policy.create_due()
def create_reminders(self):
for due in Due.objects.all():
due.create_reminder()
def do(self):
self.create_dues()
self.create_reminders()
reminders = []
for reminder in Reminder.objects.all():
if reminder.reminder_date <= datetime.date.today() and \
reminder.due.premium_paid is False and \
reminder.reminder_sent is False:
reminders.append(reminder)
send_reminder_mail(reminders)
class AdvisorRejectBidJob(CronJobBase):
"""Эдвайзер отклонил заявку студента на регистрацию на дисциплины"""
RUN_EVERY_MINS = 1
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'crop_app.advisor_reject'
def do(self):
mail_subject = 'Ваша заявка отклонена'
tasks = models.AdvisorRejectedBidTask.objects.filter(
is_success=False, )
for task in tasks:
study_plan = task.study_plan
student = study_plan.student
advisor = study_plan.advisor
msg_plain = render_to_string(
'emails/advisor_reject/advisor_rejected_bid.txt', {
'advisor_name': advisor.full_name,
'comment': task.comment
})
msg_html = render_to_string(
'emails/advisor_reject/advisor_rejected_bid.html', {
'advisor_name': advisor.full_name,
'comment': task.comment
})
if student.notify_me_from_email:
send_mail(
mail_subject,
msg_plain,
'*****@*****.**',
[student.email],
html_message=msg_html,
)
task.is_success = True
task.save()
class EmailNotifications(CronJobBase):
RUN_EVERY_MINS = 60*12
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = "core.EmailNotifications"
def do(self):
people_with_notifications = People.objects.filter(notifications__is_read=False).exclude(meta_data__mute_notifications=True).distinct()
project = get_object_or_404(Project, pk=1)
url_project = project.get_website()
for people in people_with_notifications:
messages = Notification.objects.filter(people=people, is_read=False).order_by("record", "-id")
print("Sending " + str(messages.count()) + " notifications to " + str(people))
context = {
"list": messages,
"firstname": people.name,
"url": url_project,
"organization_name": "Metabolism of Cities",
"email": people.email,
}
msg_html = render_to_string("mailbody/notifications.html", context)
msg_plain = render_to_string("mailbody/notifications.txt", context)
sender = "Metabolism of Cities" + '<*****@*****.**>'
recipient = '"' + people.name + '" <' + people.email + '>'
send_mail(
"Your latest notifications from Metabolism of Cities",
msg_plain,
sender,
[people.email],
html_message=msg_html,
)
messages.update(is_read=True)
class UpdateOverallLeaderBoardCronJob(CronJobBase):
"""
This class stores the users in descending order of their overall scores.
While displaying overall leaderboard top 10 users are
selected from the table.
"""
RUN_EVERY_MINS = 1440 # run every day
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'bhagirath.translation.update_overall_leaderboard_cron_job' # a unique code
def job(self):
user = UserProfile.objects.order_by('-overall_score')
count = UserProfile.objects.all().count()
entries = OverallLeaderboard.objects.all().count()
if entries > 0:
over = OverallLeaderboard.objects.all().delete()
i = 0
while i < count:
over = OverallLeaderboard()
over.username = user[i].user
over.overall_points_earned = user[i].overall_score
over.save()
i += 1
class insert_data_cron(CronJobBase):
"""
Send an email with the user count.
"""
RUN_EVERY_MINS = 5 #5minutes
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'cron.insert_data_cron'
def do(self):
data = csv.reader(open('C:/Users/Yesha/website/test.csv'),
delimiter=",")
for row in data:
if row[0] != 'site':
rs = ReviewSites()
rs.site = row[0]
rs.review = row[1]
rs.month = row[2]
rs.day = row[3]
rs.year = row[4]
rs.rate = row[5]
rs.sentiment = row[6]
rs.save()
class GetWeeklyStatistic(CronJobBase):
"""
Scheduled job to get data from github
"""
RUN_EVERY_MINS = 1
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'cc_github.get_weekly_statistic'
def do(self) -> None:
"""
Updates or creates github weekly statistic data for active repositories
:return: None
"""
repositories_data = get_github_statistics()
last_week_index = -1
for repository in repositories_data:
if repository:
for data in repository:
contributor = data['author']['login']
repository_id = data['repository_id']
week = date.fromtimestamp(data['weeks'][last_week_index]['w'])
additions = data['weeks'][last_week_index]['a']
deletions = data['weeks'][last_week_index]['d']
commits = data['weeks'][last_week_index]['c']
WeeklyStatistic.objects.update_or_create(repository_id=repository_id,
contributor=contributor,
week=week,
defaults={"commits": commits,
"additions": additions,
'deletions': deletions})
class StocksRateUpdaterJob(CommonCronJob):
RUN_EVERY_MINS = 0 # every 2 hours
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'nse.stocks_rate_updater'
def do(self):
CommonCronJob.logger.info("Going to run stock rate updater cron")
cnt = 0
stocks = Stock.objects.all()
currency = Currency.objects.filter(is_default=True).first()
nse = Nse()
try:
with transaction.atomic():
models_to_create = []
for stock in stocks:
cnt += 1
try:
quote = nse.get_quote(stock.code)
time.sleep(.300)
stock_rate = StockRate(code=stock.code, stock=stock,
rate=quote['basePrice'], currency=currency,
last_updated_on=timezone.now())
models_to_create.append(stock_rate)
CommonCronJob.logger.info("Completed for {0} - {1}".format(cnt, stock))
except Exception as er:
CommonCronJob.logger.error("Error for {0} - {1} - {2}".format(cnt, stock, er))
StockRate.objects.all().delete()
StockRate.objects.bulk_create(models_to_create)
CommonCronJob.logger.info("Completed stock rate updater cron")
except URLError:
CommonCronJob.logger.error("Unable to connect to nse")
except DatabaseError:
CommonCronJob.logger.error("Database error")
except Exception as e:
CommonCronJob.logger.error("Error happened")
class CheckOverdueReturnsCronJob(CronJobBase):
RUN_AT_TIMES = ['00:00']
OVERDUE_DAYS_FIRST_REMINDER = 1
OVERDUE_DAYS_SECOND_REMINDER = 7
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'returns.check_overdue_returns'
def do(self):
# first reminder that is sent to licence holder
due_date = date.today() - timedelta(
days=self.OVERDUE_DAYS_FIRST_REMINDER)
for ret in Return.objects.filter(due_date=due_date).exclude(
status__in=['submitted', 'accepted']):
send_return_overdue_email_notification(ret)
# second notice that is sent to staff
due_date = date.today() - timedelta(
days=self.OVERDUE_DAYS_SECOND_REMINDER)
for ret in Return.objects.filter(due_date=due_date).exclude(
status__in=['submitted', 'accepted']):
send_return_overdue_staff_email_notification(ret)
class RefreshCorpora(CronJobBase):
"""Scheduler for periodically refreshing twitter Corpora"""
RUN_EVERY_MINS = 360 # every six hours
schedule = Schedule(run_every_mins = RUN_EVERY_MINS)
code = 'sources.refresh_corpora' # a unique code
def do(self):
print("Examining Twitter Corpora...")
# Get all the twitter Corpora
corpus_queue = Corpus.objects.filter(variety__exact='TW')
queue_len = len(corpus_queue)
print("Created Corpus queue, length", queue_len)
# Examine each one to check for freshness. Freshen if necessary
for corpus in corpus_queue:
try:
print('examining', corpus.title)
freshen_corpus(corpus)
except:
e = sys.exc_info()[0]
print('Error', e)
class PullSongs(CronJobBase):
RUN_EVERY_MINUTES = 1
schedule = Schedule(run_every_mins = RUN_EVERY_MINUTES)
code = 'rap.PullSongs'
def do(self):
apiKey = "AIzaSyCWDfJdQfcOeDGjvL4Rs0sNnaTY0CcBMRY"
getChannelID = "https://www.googleapis.com/youtube/v3/channels?key=AIzaSyCWDfJdQfcOeDGjvL4Rs0sNnaTY0CcBMRY&forUsername=allrapnation&part=id"
channelRequest = urllib2.urlopen(getChannelID)
channel = channelRequest.read()
channelJson = json.loads(channel)
channelID = channelJson['items'][0]['id']
getPlaylistID = "https://www.googleapis.com/youtube/v3/playlists?part=snippet&channelId=" + channelID + "&key=AIzaSyCWDfJdQfcOeDGjvL4Rs0sNnaTY0CcBMRY"
playlistRequest = urllib2.urlopen(getPlaylistID)
playlist = playlistRequest.read()
playlistJson = json.loads(playlist)
for item in playlistJson['items']:
getVideosID = "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&playlistId=" + str(item['id']) +"&maxResults=50&key=AIzaSyCWDfJdQfcOeDGjvL4Rs0sNnaTY0CcBMRY"
videosRequest = urllib2.urlopen(getVideosID)
videos = videosRequest.read()
videosJson = json.loads(videos)
for songItem in videosJson['items']:
print songItem['snippet']['title']
print "Hello"
song = Song()
song.song_name = songItem['snippet']['title']
song.url = songItem['snippet']['resourceId']['videoId']
Song.save(song)
class MyCronJob(CronJobBase):
RUN_EVERY_MINS = 30 # every 30 mins
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'itieau_app.cons' # a unique code
def do(self):
gateway = SmsGateway()
gateway.loginDetails('*****@*****.**', 'r33b00ts')
url = Url.objects.filter(user__username = '******')
for url_obj in url:
r = requests.get("http://107.170.192.206/" + url_obj.script)
text = r.text.encode('ASCII', 'ignore')
split_text = str.split(text, 'Bornier')
value = split_text[url_obj.bornierSpilt][-5:]
value_str = str.strip(value, '\x1e')
value_float = float(value_str)
message = 'Water level at ' + url_obj.name +' is less then 1m!'
if value_float < 1:
contact_obj = contact.objects.filter(user__username = '******')
for c_ob in contact_obj:
number = c_ob.number
gateway.sendMessageToNumber(number, message, '8659')
class BaseCron(CronJobBase):
RUN_EVERY_MINS = 0
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
def is_running(self):
"""Checks to see if an instance of this class is already running."""
try:
running = check_output(["pgrep", "-f", self.__class__.__name__]).decode("utf-8").strip().split("\n")
if len(running) > 2:
return True
else:
return False
except CalledProcessError:
return False
def do(self):
if self.is_running():
return
start = datetime.now()
source = [s[1] for s in FetchRun.SOURCE_CHOICES if s[0] == self.fetcher.source][0]
print("Export of {} {} records from {} started at {}".format(
self.object_status, self.object_type, source, start))
out = self.fetcher().fetch(self.object_status, self.object_type)
end = datetime.now()
fetch_run = FetchRun.objects.filter(
status=FetchRun.FINISHED,
source=self.fetcher.source,
object_type=self.object_type,
object_status=self.object_status).order_by("-end_time")[0]
print("{} records exported in {}".format(out, end - start))
if fetch_run.error_count:
print("{} errors".format(fetch_run.error_count))
for e in fetch_run.errors:
print(" {}".format(e.message))
print("Export of {} {} records from {} complete at {}\n".format(
self.object_status, self.object_type, source, end))
class deleteOldJobs(CronJobBase):
RUN_EVERY_MINS = 4000
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'tools.cron.deleteOldJobs'
def do(self):
try:
#print("checking for old jobs")
how_many_days = 365
oldJobs = Job.objects.filter(start_time__lte=datetime.now() -
timedelta(days=how_many_days))
for thisjob in oldJobs:
print("deleting job for user " + str(thisjob.user) +
", job id: " + str(thisjob.id) + ", started on " +
str(thisjob.start_time) + ", file: " +
str(thisjob.output))
try:
deleteJob(thisjob.user, thisjob.id)
except OSError:
print("failed to remove output file for job " +
str(thisjob.id) + ": " + str(thisjob.output))
except:
print("error in tools.cron.deleteOldJobs")
print(traceback.format_exc())
class EpisodeToonUpdate(CronJobBase):
RUN_AT_TIMES = ['10:10', '17:00']
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'webtoon.test'
def do(self):
favorites = Webtoon.objects.filter(favorite=True)
if favorites.exists():
for webtoon in favorites:
scraper = get_episode_scraper(webtoon.site.name)
soup = scraper.get_page_soup(webtoon.toon_id)
lastest = scraper.get_lastest_episode(soup)
lastest['webtoon'] = webtoon
try:
episode, _ = WebtoonEpisodes.objects.update_or_create(
**lastest)
toon_list = get_toon_img_list(webtoon.toon_id, episode,
webtoon.site.name)
WebtoonEpisodeToon.objects.bulk_create(toon_list)
print(webtoon)
except Error:
# @todo use Logger to save error
print(Error)
class ScheduleMailingCronJob(CronJobBase):
RUN_EVERY_MINS = 1
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'mailing.schedule_mailing_cron_job'
def do(self):
unsent_mailings = Mailing.objects.filter(waiting_to_be_sent=True, )
for mailing in unsent_mailings:
data = mailing.mailchimp_list.get_mailchimp_data()
if data['stats']['member_count'] > 0:
schedule_response = mailing.schedule_using_mailchimp()
# Mailchimp does not respond on success
# so we assume it is scheduled on this scenario
if not schedule_response:
mailing.waiting_to_be_sent = False
mailing.save()
continue
logging.error(
"Error while scheduling campaign on mailchimp. " +
"Response: {}. Response status: {}.".format(
schedule_response, schedule_response['status']))
class ReportMessaggi(CronJobBase):
schedule = Schedule(run_every_mins=0)
code = 'messaggi.ReportMessaggi'
def do(self):
profiles = Profile.objects.all()
for profile in profiles:
if profile.temporizzazione_giorni > 0 and (
datetime.today() - profile.temporizzazione_data
).days > profile.temporizzazione_giorni:
count = 0
chats = Chat.objects.filter(members__in=[profile.id])
for chat in chats:
count = count + Message.objects.filter(
chat=chat.id,
is_readed=False).exclude(author=profile).count()
subject = "Hai dei nuovi messaggi"
message = "Hai ricevuto " + str(count) + " nuovi messaggi"
profile.user.email_user(subject, message)
profile.temporizzazione_data = datetime.today()
profile.save()
class ReminderJob(KuCronJob):
RUN_AT_TIMES = ['01:00']
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'kubooking.reminders'
description = "sends reminder emails"
def run(self):
emailtemplatetypes = [
EmailTemplateType.notity_all__booking_reminder,
EmailTemplateType.notify_guest_reminder
]
autosends = list(
VisitAutosend.objects.filter(
enabled=True,
template_type__in=emailtemplatetypes,
days__isnull=False,
inherit=False,
visit__eventtime__start__isnull=False,
visit__eventtime__start__gte=timezone.now()))
print "Found %d enabled autosends" % len(autosends)
inheriting_autosends = list(
VisitAutosend.objects.filter(
inherit=True,
template_type__in=emailtemplatetypes,
visit__eventtime__start__isnull=False,
visit__eventtime__start__gte=timezone.now()).all())
extra = []
for autosend in inheriting_autosends:
inherited = autosend.get_inherited()
if inherited is not None and \
inherited.enabled and \
inherited.days is not None:
autosend.days = inherited.days
autosend.enabled = inherited.enabled
extra.append(autosend)
print "Found %d enabled inheriting autosends" % len(extra)
autosends.extend(extra)
if len(autosends) > 0:
today = date.today()
print "Today is: %s" % unicode(today)
for autosend in autosends:
if autosend is not None:
print "Autosend %d for Visit %d:" % \
(autosend.id, autosend.visit.id)
start = autosend.visit.eventtime.start
if start is not None:
print " Visit starts on %s" % unicode(start)
reminderday = start.date() - timedelta(autosend.days)
print " Autosend specifies to send %d days prior," \
" on %s" % (autosend.days, reminderday)
if reminderday == today:
print " That's today; send reminder now"
autosend.visit.autosend(autosend.template_type)
else:
print " That's not today. Not sending reminder"
else:
print " Visit has no start date"
class EvaluationReminderJob(KuCronJob):
RUN_AT_TIMES = ['02:00']
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'kubooking.evaluationreminder'
description = "sends evaluation reminder emails"
days = 5
def run(self):
emailtemplate = [
EmailTemplateType.notify_guest__evaluation_second,
EmailTemplateType.notify_guest__evaluation_second_students
]
filter = {
'template_type__in': emailtemplate,
}
autosends = list(
VisitAutosend.objects.filter(
inherit=False,
enabled=True,
days__isnull=False,
).filter(**filter).all())
print "Found %d enabled autosends" % len(autosends)
inheriting_autosends = list(
VisitAutosend.objects.filter(
inherit=True, ).filter(**filter).all())
extra = []
for autosend in inheriting_autosends:
inherited = autosend.get_inherited()
if inherited is not None and inherited.enabled:
autosend.enabled = inherited.enabled
extra.append(autosend)
print "Found %d enabled inheriting autosends" % len(extra)
autosends.extend(extra)
try:
if len(autosends):
today = date.today()
print "Today is: %s" % unicode(today)
for autosend in autosends:
visit = autosend.visit
print "Autosend %d for Visit %d:" % \
(autosend.id, visit.id)
if visit.end_datetime is None:
print "Visit %d has no apparent end_datetime" %\
visit.id
else:
print " Visit ends on %s" % \
unicode(visit.end_datetime.date())
alertday = visit.end_datetime.date() + \
timedelta(self.days)
print " Hardcoded to send %d days after " \
"completion, on %s" % (self.days, alertday)
if alertday == today:
print " That's today; sending messages now"
product = visit.product
if product is not None:
evals = product.surveyxactevaluation_set.all()
for evaluation in evals:
evaluation.send_second_notification(visit)
else:
print " That's not today. Not sending messages"
finally:
for autosend in autosends:
autosend.refresh_from_db()
class IdleHostroleJob(KuCronJob):
RUN_AT_TIMES = ['01:00']
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'kubooking.idlehost'
description = "sends notification emails regarding idle host roles"
def run(self):
visit_qs = Visit.objects.annotate(
num_bookings=Count('bookings'), ).filter(
num_bookings__gt=0,
workflow_status__in=[
Visit.WORKFLOW_STATUS_BEING_PLANNED,
Visit.WORKFLOW_STATUS_REJECTED,
Visit.WORKFLOW_STATUS_AUTOASSIGN_FAILED
])
visits_needing_hosts = [
visit for visit in visit_qs if visit.needs_hosts
]
autosends = list(
VisitAutosend.objects.filter(
enabled=True,
template_type=EmailTemplateType.notify_host__hostrole_idle,
days__isnull=False,
inherit=False,
visit__in=visits_needing_hosts).all())
print "Found %d enabled autosends" % len(autosends)
inheriting_autosends = list(
VisitAutosend.objects.filter(
inherit=True,
template_type=EmailTemplateType.notify_host__hostrole_idle,
visit__in=visits_needing_hosts).all())
extra = []
for autosend in inheriting_autosends:
inherited = autosend.get_inherited()
if inherited is not None and \
inherited.enabled and \
inherited.days is not None:
autosend.days = inherited.days
autosend.enabled = inherited.enabled
extra.append(autosend)
print "Found %d enabled inheriting autosends" % len(extra)
autosends.extend(extra)
try:
if len(autosends) > 0:
today = date.today()
print "Today is: %s" % unicode(today)
for autosend in autosends:
if autosend is None:
continue
print "Autosend %d for Visit %d:" % \
(autosend.id, autosend.visit.id)
first_booking = autosend.visit.\
bookings.earliest('statistics__created_time')
print " Visit has its first booking on %s" % \
unicode(
first_booking.statistics.created_time.date()
)
alertday = first_booking.statistics.created_time.\
date() + timedelta(autosend.days)
print " Autosend specifies to send %d days after " \
"first booking, on %s" % (autosend.days, alertday)
if alertday == today:
print " That's today; send alert now"
try:
autosend.visit.autosend(
EmailTemplateType.notify_host__hostrole_idle)
except Exception as e:
print e
else:
print " That's not today. Not sending alert"
finally:
for autosend in autosends:
autosend.refresh_from_db()
class SurveyObs(CronJobBase):
RUN_EVERY_MINS = 0.1
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'YSE_App.data_ingest.YSE_observations.SurveyObs'
def do(self):
parser = self.add_options(usage='')
options, args = parser.parse_known_args()
try:
config = configparser.ConfigParser()
config.read("%s/settings.ini"%djangoSettings.PROJECT_DIR)
parser = self.add_options(usage='',config=config)
options, args = parser.parse_known_args()
self.options = options
if os.path.exists('%s/surveyfields.txt'%djangoSettings.STATIC_ROOT):
self.add_survey_fields('%s/surveyfields.txt'%djangoSettings.STATIC_ROOT)
uploaddict = self.process_emails()
except Exception as e:
print(e)
exc_type, exc_obj, exc_tb = sys.exc_info()
print('line number %i'%exc_tb.tb_lineno)
if uploaddict is not None: self.upload(uploaddict)
def process_emails(self):
body = ""
html = ""
survey_fields = {}
########################################################
# Get All Email
########################################################
mail = imaplib.IMAP4_SSL('imap.gmail.com', 993) #, ssl_context=ctx
## NOTE: This is not the way to do this. You will want to implement an industry-standard login step ##
mail.login(self.options.SMTP_LOGIN, self.options.SMTP_PASSWORD)
mail.select('YSE', readonly=False)
retcode, msg_ids_bytes = mail.search(None, '(UNSEEN)')
msg_ids = msg_ids_bytes[0].decode("utf-8").split(" ")
try:
if retcode != "OK" or msg_ids[0] == "":
raise ValueError("No messages")
except ValueError as err:
print("%s. Exiting..." % err.args)
mail.close()
mail.logout()
del mail
print("Process done.")
return None
objs,ras,decs = [],[],[]
print('length of msg_ids %s' %len(msg_ids))
for i in range(len(msg_ids)):
########################################################
# Iterate Over Email
########################################################
typ, data = mail.fetch(msg_ids[i],'(RFC822)')
msg = email.message_from_bytes(data[0][1])
if msg.is_multipart():
for part in msg.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if (ctype == 'text/plain' or ctype == 'text/html') and 'attachment' not in cdispo:
body = part.get_payload(decode=True).decode('utf-8') # decode
break
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = msg.get_payload(decode=True).decode('utf-8')
header = None
for line in body.split('\n'):
lineparts = line.replace('\r','').replace('> ','').split('\t')
if 'exp_name' in line:
header = lineparts
elif 'YSE' in line and header is not None:
survey_dict = {}
for h,l in zip(header,lineparts):
if h in ingest_keys_map.keys():
if ingest_keys_map[h][1] is None:
survey_dict[ingest_keys_map[h][0]] = l
else:
if ingest_keys_map[h][0] != 'photometric_band':
survey_dict[ingest_keys_map[h][0]] = ingest_keys_map[h][1](l)
else:
survey_dict[ingest_keys_map[h][0]] = ingest_keys_map[h][1](
l,np.array(lineparts)[np.array(header) == 'exp_name'][0])
survey_dict['status'] = 'Successful'
survey_fields[survey_dict['image_id']] = survey_dict
# Mark messages as "Seen"
result, wdata = mail.store(msg_ids[i], '+FLAGS', '\\Seen')
# manually setting MSB name
for k in survey_fields.keys():
survey_fields[k]['msb'] = survey_fields[k]['survey_field'].split('.')[0]
return survey_fields
def add_survey_fields(self,surveyfile):
fid,ra,dec = np.loadtxt(surveyfile,unpack=True,dtype=str)
SurveyUploadDict = {}
for f,r,d in zip(fid,ra,dec):
sc = SkyCoord(r,d,unit=(u.hourangle,u.deg))
SurveyUploadDict[f] = {'obs_group':'YSE',
'field_id':f,
'cadence':3,
'instrument':'GPC1',
'ztf_field_id':f.split('.')[0],
'ra_cen':sc.ra.deg,
'dec_cen':sc.dec.deg,
'width_deg':3.1,
'height_deg':3.1,
'active':1}
url = '%s'%self.options.dburl.replace('/api','/add_yse_survey_fields')
r = requests.post(url = url, data = json.dumps(SurveyUploadDict),
auth=HTTPBasicAuth(self.options.dblogin,self.options.dbpassword))
print('YSE_PZ says: %s'%json.loads(r.text)['message'])
def upload(self,SurveyUploadDict):
url = '%s'%self.options.dburl.replace('/api','/add_yse_survey_obs')
try:
r = requests.post(url = url, data = json.dumps(SurveyUploadDict),
auth=HTTPBasicAuth(self.options.dblogin,self.options.dbpassword))
try: print('YSE_PZ says: %s'%json.loads(r.text)['message'])
except: print(r.text)
except exception as e: print(e)
print("upload finished.")
def add_options(self, parser=None, usage=None, config=None):
import argparse
if parser == None:
parser = argparse.ArgumentParser(usage=usage, conflict_handler="resolve")
# The basics
parser.add_argument('-v', '--verbose', action="count", dest="verbose",default=1)
parser.add_argument('--clobber', default=False, action="store_true",
help='clobber output file')
#parser.add_option('-s','--settingsfile', default=None, type=str,
# help='settings file (login/password info)')
if config:
parser.add_argument('--dblogin', default=config.get('main','dblogin'), type=str,
help='database login, if post=True (default=%default)')
parser.add_argument('--dbemail', default=config.get('main','dbemail'), type=str,
help='database login, if post=True (default=%default)')
parser.add_argument('--dbpassword', default=config.get('main','dbpassword'), type=str,
help='database password, if post=True (default=%default)')
parser.add_argument('--dburl', default=config.get('main','dburl'), type=str,
help='URL to POST transients to a database (default=%default)')
parser.add_argument('--SMTP_LOGIN', default=config.get('YSE_SMTP_provider','SMTP_LOGIN'), type=str,
help='SMTP login (default=%default)')
parser.add_argument('--SMTP_PASSWORD', default=config.get('YSE_SMTP_provider','SMTP_PASSWORD'), type=str,
help='SMTP password (default=%default)')
parser.add_argument('--SMTP_HOST', default=config.get('YSE_SMTP_provider','SMTP_HOST'), type=str,
help='SMTP host (default=%default)')
parser.add_argument('--SMTP_PORT', default=config.get('YSE_SMTP_provider','SMTP_PORT'), type=str,
help='SMTP port (default=%default)')
return parser
class ReadFromLuftdateInfoJSON(CronJobBase):
RUN_EVERY_MINS = 5 # every 2 hours
schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
code = 'dim4webapp.readFromLDIJason' # a unique code
def do(self):
url = r'http://api.luftdaten.info/static/v2/data.json'
with urllib.request.urlopen(url) as urlJson:
data = json.loads(urlJson.read())
ownerLDI, created = Owner.objects.get_or_create(name='LufdatenInfo',
idNumber=int(1))
if created:
ownerLDI.save()
print(ownerLDI)
usedIDSet = set()
for sensorData in data:
sensorLatitude = sensorData['location']['latitude']
sensorLongitude = sensorData['location']['longitude']
currentSensor = Sensor.objects.filter(
ldi_number=int(sensorData['sensor']['id']))
type_selection = [
d['value_type'] for d in sensorData['sensordatavalues'][:]
] == "P1"
if currentSensor.count() == 0:
print('new')
print(int(sensorData['id']))
sensorModel, created = Sensor.objects.get_or_create(
owner_id=ownerLDI.id,
location_number=str(sensorData['location']['id']),
ldi_number=int(sensorData['sensor']['id']),
geom=Point(float(sensorData['location']['longitude']),
float(sensorData['location']['latitude'])),
location='Empty',
longitude=float(sensorData['location']['longitude']),
latitude=float(sensorData['location']['latitude']),
)
geoNamesRequestUrl = r'http://api.geonames.org/findNearbyJSON?lat=' + str(
sensorLatitude) + '&lng=' + str(
sensorLongitude) + '&username=DrKarlMarxx'
try:
with urllib.request.urlopen(geoNamesRequestUrl) as urlJson:
geoNamesData = json.loads(urlJson.read())
sensorModel.location = geoNamesData["geonames"][0][
"toponymName"]
except:
pass
sensorModel.save()
else:
sensorModel = currentSensor[0]
if sensorModel.id not in usedIDSet:
for sensorvalueList in sensorData['sensordatavalues'][:]:
typeFromJson = str(sensorvalueList['value_type'])
valueFromJson = float(sensorvalueList['value'])
timestepData = SensorValue(sensor_id=sensorModel.id,
value=valueFromJson,
type=typeFromJson)
timestepData.save()
runningCurrentSensorValue = CurrentSensorValue.objects.filter(
sensor_id=sensorModel.id, type=typeFromJson)
if runningCurrentSensorValue.count() == 0:
currentSensorValue, created = CurrentSensorValue.objects.get_or_create(
sensor_id=sensorModel.id,
value=valueFromJson,
type=typeFromJson)
else:
currentSensorValue = runningCurrentSensorValue[0]
currentSensorValue.value = valueFromJson
currentSensorValue.save()
"""
weatherDataRequestUrl = r'api.openweathermap.org/data/2.5/weather?lat='+str(sensorLatitude)+'&lon='+str(sensorLongitude)
with urllib.request.urlopen(weatherDataRequestUrl) as urlJson:
weatherData = json.loads(urlJson.read())
weatherData, created = WeatherData.objects.get_or_create(sensorvalue_id=timestepData.id,
temperature=weatherData['main']['temp'],
humidity = weatherData['main']['humidity'],
pressure = weatherData['main']['pressure'],
temp_min = weatherData['main']['temp_min'],
temp_max = weatherData['main']['temp_max'],
wind_speed = weatherData['wind']['speed'],
wind_deg = weatherData['wind']['deg'],
rain = weatherData['rain']['3h'],
clouds = weatherData['clouds']['all'],
name = weatherData['name'])
"""
usedIDSet.add(sensorModel.id)
for savedSensorValue in SensorValue._meta.get_fields():
if (timezone.now() - savedSensorValue.created).days >= 7:
savedSensorValue.delete()
