class Automation(object):
def __init__(self):
pass
@huey.periodic_task(crontab(minute='*', hour='*'))
def cleanUserDatabase():
'''Huey task to routinely clean the database of all user accounts that are unverified ''' # Set to Five Minutes for debug
conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back)
# expired = time.time() - 86400 # One day ago
expired = time.time() - 300 # Five minute(s) ago
r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up
result = r.table(c.user_table).filter((r.row['created'] < expired) and (r.row['verified'] == False))['id'].coerce_to('array').run(conn)
# The above string asks the database server to locate all items out of the user table where 'created' is older than 'expired' and verified is false
if len(result) > 0:
database.delete(c.user_table, conn, database.args(result)) # Delete all the IDs recieved by the query string out of the 'user' database
database.delete(c.auth_table, conn, database.args(result)) # Delete all the IDs recieved by the query string out of the 'auth' database
print "Removed {} unverified users from the database".format(len(result))
conn.close() # Clean up the database connection
@huey.periodic_task(crontab(minute='*', hour='*'))
def cleanLoginSessions():
'''Huey task to clear login sessions out of the database'''
conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back)
expired = time.time() - 120 # Two minute(s) ago
r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up
result = r.table(c.auth_table).update(lambda row:{'login_sessions':row['login_sessions'].filter(lambda session:session['timestamp'] > expired)}).run(conn)
# The above string asks the database server to find all login sessions in the table 'auth' with a timestamp older than 'expired' and delete them
# This is a heavy action - should be stress tested
if result['replaced'] > 0:
print "Removed expired login_sessions for {} users from the database".format(result['replaced'])
conn.close() # Clean up the database connection
def test_crontab_hour(self):
# validates the following hours
valids = [0, 1, 4, 6, 8, 9, 12, 18]
validate_h = crontab(hour='8-9,*/6,1,4')
for x in range(24):
res = validate_h(datetime.datetime(2011, 1, 1, x))
self.assertEqual(res, x in valids)
edge = crontab(hour=0)
self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0)))
self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0)))
def test_crontab_hour(self):
# validates the following hours
valids = [0, 1, 4, 6, 8, 9, 12, 18]
validate_h = crontab(hour='8-9,*/6,1,4')
for x in range(24):
res = validate_h(datetime.datetime(2011, 1, 1, x))
self.assertEqual(res, x in valids)
edge = crontab(hour=0)
self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0)))
self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0)))
def set_index_crontab(value: str, conn: Connection) -> None:
"""
:raises InvalidConfig: If the crontab is syntactically invalid.
"""
try:
crontab(**parse_crontab(value), strict=True)
except ValueError:
raise InvalidConfig(f"{value} is not a valid crontab.")
conn.execute(
"UPDATE system__config SET value = ? WHERE key = ?",
(value, INDEX_CRONTAB),
)
def test_crontab_minute(self):
# validates the following minutes
valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54]
validate_m = crontab(minute='4,8-9,*/6,1')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
# We don't ensure *every* X minutes, but just on the given intervals.
valids = [0, 16, 32, 48]
validate_m = crontab(minute='*/16')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_minute(self):
# validates the following minutes
valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54]
validate_m = crontab(minute='4,8-9,*/6,1')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
# We don't ensure *every* X minutes, but just on the given intervals.
valids = [0, 16, 32, 48]
validate_m = crontab(minute='*/16')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_all_together(self):
# jan 1, 2011 is a saturday
# may 1, 2011 is a sunday
validate = crontab(month='1,5',
day='1,4,7',
day_of_week='0,6',
hour='*/4',
minute='1-5,10-15,50')
self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11)))
self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50)))
self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1)))
# fails validation on month
self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11)))
# fails validation on day
self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11)))
# fails validation on day_of_week
self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11)))
# fails validation on hour
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11)))
# fails validation on minute
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))
def test_crontab_all_together(self):
# jan 1, 2011 is a saturday
# may 1, 2011 is a sunday
validate = crontab(
month='1,5',
day='1,4,7',
day_of_week='0,6',
hour='*/4',
minute='1-5,10-15,50'
)
self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11)))
self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50)))
self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1)))
# fails validation on month
self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11)))
# fails validation on day
self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11)))
# fails validation on day_of_week
self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11)))
# fails validation on hour
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11)))
# fails validation on minute
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))
def test_crontab_month(self):
# validates the following months, 1, 4, 7, 8, 9
valids = [1, 4, 7, 8, 9]
validate_m = crontab(month='1,4,*/6,8-9')
for x in range(1, 13):
res = validate_m(datetime.datetime(2011, x, 1))
self.assertEqual(res, x in valids)
def test_crontab_month(self):
# validates the following months, 1, 4, 7, 8, 9
valids = [1, 4, 7, 8, 9]
validate_m = crontab(month='1,4,*/6,8-9')
for x in range(1, 13):
res = validate_m(datetime.datetime(2011, x, 1))
self.assertEqual(res, x in valids)
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
def test_crontab_day(self):
# validates the following days
valids = [1, 4, 7, 8, 9, 13, 19, 25, 31]
validate_d = crontab(day='*/6,1,4,8-9')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
def schedule_subscriptions_check(user, minutes='0', hours='12'):
def wrapper():
subscriptions_check(user)
schedule = crontab(minute=minutes, hour=hours)
task_name = f'user_{user.id}_subscriptions'
db_periodic_task(schedule, name=task_name)(wrapper)
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
def test_crontab_minute(self):
# validates the following minutes
valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54]
validate_m = crontab(minute='4,8-9,*/6,1')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_day_of_week(self):
# validates the following days of week
# jan, 1, 2011 is a saturday
valids = [2, 4, 9, 11, 16, 18, 23, 25, 30]
validate_dow = crontab(day_of_week='0,2')
for x in range(1, 32):
res = validate_dow(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_sunday(self):
for dow in ('0', '7'):
validate = crontab(day_of_week=dow, hour='0', minute='0')
valid = set((2, 9, 16, 23, 30))
for x in range(1, 32):
if x in valid:
self.assertTrue(validate(datetime.datetime(2011, 1, x)))
else:
self.assertFalse(validate(datetime.datetime(2011, 1, x)))
def test_crontab_sunday(self):
for dow in ('0', '7'):
validate = crontab(day_of_week=dow, hour='0', minute='0')
valid = set((2, 9, 16, 23, 30))
for x in range(1, 32):
if x in valid:
self.assertTrue(validate(datetime.datetime(2011, 1, x)))
else:
self.assertFalse(validate(datetime.datetime(2011, 1, x)))
def test_crontab_day_of_week(self):
# validates the following days of week
# jan, 1, 2011 is a saturday
valids = [2, 4, 9, 11, 16, 18, 23, 25, 30]
validate_dow = crontab(day_of_week='0,2')
for x in range(1, 32):
res = validate_dow(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
def schedule(action):
cfg = app.config.get("HUEY_SCHEDULE", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError(
"No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again."
.format(x=action))
return crontab(**action_cfg)
def test_crontab_day(self):
# validates the following days
valids = [1, 4, 7, 8, 9, 13, 19, 25, 31]
validate_d = crontab(day='*/6,1,4,8-9')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
valids = [1, 11, 21, 31]
validate_d = crontab(day='*/10')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
valids.pop() # Remove 31, as feb only has 28 days.
for x in range(1, 29):
res = validate_d(datetime.datetime(2011, 2, x))
self.assertEqual(res, x in valids)
def test_crontab_day(self):
# validates the following days
valids = [1, 4, 7, 8, 9, 13, 19, 25, 31]
validate_d = crontab(day='*/6,1,4,8-9')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
valids = [1, 11, 21, 31]
validate_d = crontab(day='*/10')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
valids.pop() # Remove 31, as feb only has 28 days.
for x in range(1, 29):
res = validate_d(datetime.datetime(2011, 2, x))
self.assertEqual(res, x in valids)
def schedule_alarm(alarm_name, cron_minutes, cron_hours, cron_days):
def wrapper():
alarm(alarm_name)
# The schedule that was specified for this task.
schedule = crontab(minute=cron_minutes,
hour=cron_hours,
day_of_week=cron_days)
periodic_task(schedule, name=alarm_name)(wrapper)
def load_periodic_tasks(self):
"""
This method read schedule.info file from the instance. At the same folder level that manage.py file.
:return:
"""
schedule_process = self.execute_command('makeschedule')
while self.is_running(schedule_process):
time.sleep(0.5)
if os.path.isfile(os.path.join(os.path.dirname(self.script_path), 'schedule.info')):
self._logger.debug('Schedule info created')
else:
self._logger.debug('Schedule info not found')
return
info_file = os.path.join(os.path.dirname(self.script_path), 'schedule.info')
with open(info_file, 'r') as f:
lines = f.readlines()
for ln in lines:
ln = ln.rstrip().strip().rstrip()
if ln.startswith(';') or ln.startswith('#'):
continue
info = ln.split()
if len(info) == 6:
self._logger.info('Added periodic method: %s', ln)
self.periodic_tasks.append({
'method': info[5],
'validate_datetime': crontab(
minute=info[0],
hour=info[1],
day_of_week=info[2],
day=info[3],
month=info[4])
})
elif len(info) > 1:
self._logger.info('Invalid cron line.')
if len(self.periodic_tasks) == 0:
self._logger.info('No periodic task found')
from huey import crontab
from huey.contrib.djhuey import db_periodic_task
from influxdb_metrics.loader import write_points
from foodsaving.users.stats import get_users_stats
from foodsaving.utils import stats_utils
@db_periodic_task(crontab(hour='*/6', minute=5)) # every 6 hours
def record_user_stats():
stats_utils.periodic_task('users__record_user_stats')
fields = get_users_stats()
write_points([{
'measurement': 'karrot.users',
'fields': fields,
}])
import time
import redis
from django.conf import settings
from huey import crontab
from huey.contrib.djhuey import db_periodic_task
from .models import Log
from .main import post_blood_weather, update_blood_groups
"""
Method to fetch blood groups and update status
"""
def fetch_and_update():
blood_groups, log = update_blood_groups()
post_blood_weather(blood_groups, log)
"""
Periodic task to update blood statuses
"""
@db_periodic_task(crontab(minute='*/15'))
def main_blood_groups_task():
fetch_and_update()
time.sleep(settings.BLOOD_FETCH_INTERVAL)
return True
multiprocessing.set_start_method("fork")
except RuntimeError:
# sometimes it has already been set by
# other libs
pass
redis_host = None
if settings.cache_url:
redis_host = settings.cache_url.host
huey = PriorityRedisHuey("opennem.scheduler", host=redis_host)
# export tasks
@huey.periodic_task(crontab(minute="*/5"), priority=90)
@huey.lock_task("schedule_live_tasks")
def schedule_live_tasks() -> None:
if settings.workers_run:
export_power(priority=PriorityType.live)
export_flows()
@huey.periodic_task(crontab(minute="*/30"), priority=90)
@huey.lock_task("schedule_custom_tasks")
def schedule_custom_tasks() -> None:
if settings.workers_run:
export_electricitymap()
@huey.periodic_task(crontab(hour="13"), priority=50)
from huey import RedisHuey, crontab
from scrape import get_data_send_mail
queue = RedisHuey("scrape-trending")
@queue.periodic_task(crontab(minute="0", hour="*"))
def send_mail():
get_data_send_mail()
from huey import crontab
from huey.contrib.djhuey import db_periodic_task
import logging
from danceschool.core.constants import getConstant
from .helpers import createExpenseItemsForEvents, createExpenseItemsForVenueRental,createRevenueItemsForRegistrations
# Define logger for this file
logger = logging.getLogger(__name__)
@db_periodic_task(crontab(minute='*/60'))
def updateFinancialItems():
'''
Every hour, create any necessary revenue items and expense items for
activities that need them.
'''
logger.info('Creating automatically-generated financial items.')
if getConstant('financial__autoGenerateExpensesEventStaff'):
createExpenseItemsForEvents()
if getConstant('financial__autoGenerateExpensesVenueRental'):
createExpenseItemsForVenueRental()
if getConstant('financial__autoGenerateRevenueRegistrations'):
createRevenueItemsForRegistrations()
def retry_command(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@queue_command(test_invoker, retries=3, retry_delay=10)
def retry_command_slow(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@periodic_command(test_invoker, crontab(minute='0'))
def every_hour():
state['p'] = 'y'
# create a log handler that will track messages generated by the consumer
class TestLogHandler(logging.Handler):
def __init__(self, *args, **kwargs):
self.messages = []
logging.Handler.__init__(self, *args, **kwargs)
def emit(self, record):
self.messages.append(record.getMessage())
class SkewConsumerTestCase(unittest.TestCase):
def setUp(self):
def _throw_error_task(message=None):
raise TestException(message or 'bampf')
throw_error_task = huey.task()(_throw_error_task)
throw_error_task_res = huey_results.task()(_throw_error_task)
@huey_results.task()
def add_values(a, b):
return a + b
@huey_results.task()
def add_values2(a, b):
return a + b
@huey_results.periodic_task(crontab(minute='0'))
def hourly_task2():
state['periodic'] = 2
@huey_results.task()
def returns_none():
return None
@huey_store_none.task()
def returns_none2():
return None
class BaseQueueTestCase(BaseTestCase):
def setUp(self):
global state
import sqlite3
from conreq.utils.generic import get_database_type
from django.conf import settings
from django.db import connection
from huey import crontab
from huey.contrib.djhuey import db_periodic_task
DB_ENGINE = get_database_type()
HUEY_FILENAME = getattr(settings, "HUEY_FILENAME")
@db_periodic_task(crontab(minute="0", hour="12", day="1/1"))
def bg_tasks_vacuum():
"""Periodically preforms a SQLITE vacuum on the background task database."""
with sqlite3.connect(HUEY_FILENAME) as cursor:
cursor.execute("VACUUM")
@db_periodic_task(crontab(minute="0", hour="12", day="1/1"))
def db_vacuum():
"""Periodically performs any cleanup tasks needed for the default database."""
if DB_ENGINE == "SQLITE3":
with connection.cursor() as cursor:
cursor.execute("VACUUM")
class TestException(Exception):
pass
@huey.task()
def _throw_error_task(message=None):
raise TestException(message or 'bampf')
throw_error_task = huey.task()(_throw_error_task)
throw_error_task_res = huey_results.task()(_throw_error_task)
@huey_results.task()
def add_values(a, b):
return a + b
@huey_results.periodic_task(crontab(minute='0'))
def hourly_task2():
state['periodic'] = 2
@huey_results.task()
def returns_none():
return None
@huey_store_none.task()
def returns_none2():
return None
class BaseQueueTestCase(BaseTestCase):
def setUp(self):
global state
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Survaider
from huey import crontab
from jupiter import huey
from jupiter.sentient.model import AspectQ
import datetime
@huey.periodic_task(crontab(minute='*/1'))
def spawn_processor_m_30():
"""These processors are run every 30 minutes
Processors Index
----------------
These processors are explicitly run:
- External API Calls
- (Extend this list as needed)
Developer Notes
---------------
All the tasks (such as scraping, or, machine learning etc.) must be *spawned*
from here. It is important to note that these tasks are **required** to be
non-blocking and must be huey tasks.
To prevent any magic to happen, do **not** spawn any periodic tasks inside.
"""
print ("Process Running-- delay : 2min")
# print(len(AspectQ.objects))
for obj in AspectQ.objects:
def retry_command(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.task(retries=3, retry_delay=10)
def retry_command_slow(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.periodic_task(crontab(minute='0'))
def every_hour():
state['p'] = 'y'
# Create a log handler that will track messages generated by the consumer.
class TestLogHandler(logging.Handler):
def __init__(self, *args, **kwargs):
self.messages = []
logging.Handler.__init__(self, *args, **kwargs)
def emit(self, record):
self.messages.append(record.getMessage())
class ConsumerTestCase(unittest.TestCase):
return a * b
@db_task() # Opens DB connection for duration of task.
def slow(n):
tprint('going to sleep for %s seconds' % n)
time.sleep(n)
tprint('finished sleeping for %s seconds' % n)
return n
@task(retries=1, retry_delay=5, context=True)
def flaky_task(task=None):
if task is not None and task.retries == 0:
tprint('flaky task succeeded on retry.')
return 'succeeded on retry.'
tprint('flaky task is about to raise an exception.', 31)
raise Exception('flaky task failed!')
# Periodic tasks.
@periodic_task(crontab(minute='*/2'))
def every_other_minute():
tprint('This task runs every 2 minutes.', 35)
@periodic_task(crontab(minute='*/5'))
def every_five_mins():
tprint('This task runs every 5 minutes.', 34)
send_mail(subject, body, from_, tos)
def _get_comment_body(blogitem, blogcomment):
base_url = "https://%s" % Site.objects.get_current().domain
if "peterbecom.local" in base_url:
base_url = "http://*****:*****@periodic_task(crontab(hour="*"))
def run_populate_blogitem_daily_hits():
date = timezone.now() - datetime.timedelta(days=1)
try:
sum_count, items_count = BlogItemDailyHits.rollup_date(date)
print(
"Rolled up {:,} blogitems a total of {:,} hits".format(
items_count, sum_count
)
)
except BlogItemDailyHitsExistingError:
print("Already rolled up for {}".format(date))
@db_task() # Opens DB connection for duration of task.
def slow(n):
tprint('going to sleep for %s seconds' % n)
time.sleep(n)
tprint('finished sleeping for %s seconds' % n)
return n
@task(retries=1, retry_delay=5, context=True)
def flaky_task(task=None):
if task is not None and task.retries == 0:
tprint('flaky task succeeded on retry.')
return 'succeeded on retry.'
tprint('flaky task is about to raise an exception.', 31)
raise Exception('flaky task failed!')
# Periodic tasks.
@periodic_task(crontab(minute='*/2'))
def every_other_minute():
tprint('This task runs every 2 minutes.', 35)
@periodic_task(crontab(minute='*/5'))
def every_five_mins():
tprint('This task runs every 5 minutes.', 34)
from django.conf import settings
from django.db.models import F, OuterRef, Subquery
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from huey import crontab
from huey.contrib.djhuey import db_task, db_periodic_task, HUEY
from huey.exceptions import TaskLockedException
from camp.apps.monitors.models import Entry
from camp.apps.monitors.purpleair import api
from camp.apps.monitors.purpleair.models import PurpleAir
@db_periodic_task(crontab(minute='*/2'), priority=50)
def import_recent_data():
print('[import_recent_data]')
if HUEY.pending_count() > settings.MAX_QUEUE_SIZE:
return
for monitor in PurpleAir.objects.all():
print('\n' * 10, '[import_recent_data]', monitor.name, '\n' * 10)
import_monitor_data.schedule([monitor.pk], delay=1, priority=30)
@db_task()
def import_monitor_history(monitor_id, end=None):
monitor = PurpleAir.objects.get(pk=monitor_id)
end = end or timezone.now()
feeds = monitor.get_feeds(end=end, results=1000)
res_queue = DummyQueue(res_queue_name)
res_store = DummyDataStore(res_queue_name)
res_huey = Huey(res_queue, res_store, schedule)
res_huey_nones = Huey(res_queue, res_store, store_none=True)
# store some global state
state = {}
# create a decorated queue command
@huey.task()
def add(key, value):
state[key] = value
# create a periodic queue command
@huey.periodic_task(crontab(minute='0'))
def add_on_the_hour():
state['periodic'] = 'x'
# define a command using the class
class AddTask(QueueTask):
def execute(self):
k, v = self.data
state[k] = v
# create a command that raises an exception
class BampfException(Exception):
pass
@huey.task()
def throw_error():
@huey.task()
def send_mail(content):
smtp_config = setting_col.find_one({'key': 'mail'})
receivers = [data.get('mail') for data in notice_col.find({})]
try:
if mail_notice(smtp_config, receivers, content):
logger.info('邮件发送成功')
else:
logger.critical('Error: 无法发送邮件')
except smtplib.SMTPException as error:
logger.critical('Error: 无法发送邮件 {}'.format(error))
@huey.periodic_task(crontab(minute='*/2'))
def update_rate_remain():
for account in github_col.find():
github_username = account.get('username')
github_password = account.get('password')
try:
g = Github(github_username)
github_col.update_one({'username': github_username}, {
'$set': {
'rate_remaining': int(g.get_rate_limit().search.remaining),
'rate_limit': int(g.get_rate_limit().search.limit)
}
})
except Exception as error:
logger.error(error)
def explicit_retry(k):
if k not in state:
state[k] = 'fixed'
raise RetryTask()
return state[k]
@test_huey.task(retries=1, include_task=True)
def retry_with_task(a, b, task=None):
assert task is not None
if a + b < 0:
raise RetryTask()
return a + b
@test_huey.periodic_task(crontab(minute='2'))
def hourly_task():
state['p'] = 'y'
@test_huey.periodic_task(crontab(minute='3'), retries=3)
def hourly_task2():
try:
state['p2'] += 1
except KeyError:
state['p2'] = 1
raise
@test_huey.task(retries=2)
@test_huey.lock_task('test-lock')
@test_huey.task(retries=2)
def explicit_retry(k):
if k not in state:
state[k] = 'fixed'
raise RetryTask()
return state[k]
@test_huey.task(retries=1, include_task=True)
def retry_with_task(a, b, task=None):
assert task is not None
if a + b < 0:
raise RetryTask()
return a + b
@test_huey.periodic_task(crontab(minute='2'))
def hourly_task():
state['p'] = 'y'
@test_huey.periodic_task(crontab(minute='3'), retries=3)
def hourly_task2():
try:
state['p2'] += 1
except KeyError:
state['p2'] = 1
raise
@test_huey.task(retries=2)
@test_huey.lock_task('test-lock')
def locked_task(a, b):
return a + b
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.task(retries=3, retry_delay=10)
def retry_task_delay(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.periodic_task(crontab(minute='2'))
def hourly_task():
state['p'] = 'y'
class TestExecution(HueyTestCase):
def create_consumer(self, worker_type='thread'):
return Consumer(self.huey,
max_delay=0.1,
workers=2,
worker_type=worker_type)
def test_threaded_execution(self):
consumer = self.create_consumer()
r1 = modify_state('k1', 'v1')
r2 = modify_state('k2', 'v2')
import random
import time
from huey import crontab
from config import huey
@huey.task()
def count_beans(num):
print('-- counted %s beans --' % num)
return 'Counted %s beans' % num
@huey.periodic_task(crontab(minute='*/5'))
def every_five_mins():
print('Consumer prints this every 5 mins')
@huey.task(retries=3, retry_delay=10)
def try_thrice():
if random.randint(1, 3) == 1:
print('OK')
else:
print('About to fail, will retry in 10 seconds')
raise Exception('Crap something went wrong')
@huey.task()
def slow(n):
time.sleep(n)
print('slept %s' % n)
# from app.tasks.gift_code_management import send_gift_code_winner_approval_pm
from cubersio.tasks.reddit import prepare_new_competition_notification,\
prepare_end_of_competition_info_notifications
from . import huey
# -------------------------------------------------------------------------------------------------
if app.config['IS_DEVO']:
# don't run as periodic in devo
WRAP_WEEKLY_COMP_SCHEDULE = lambda _ : False
RUN_RANKINGS_SCHEDULE = lambda _ : False
else:
# Run the rankings task several hours after the comp stuff. I think we're having memory issues.
WRAP_WEEKLY_COMP_SCHEDULE = crontab(day_of_week='1', hour='2', minute='0')
RUN_RANKINGS_SCHEDULE = crontab(day_of_week='1', hour='6', minute='0')
# -------------------------------------------------------------------------------------------------
@huey.periodic_task(RUN_RANKINGS_SCHEDULE)
def run_weekly_site_rankings():
""" A periodic task to run the site rankings stuff weekly. """
run_user_site_rankings()
@huey.task()
def run_user_site_rankings():
""" A task to run the calculations to update user site rankings based on the latest data. """
