class Automation(object): def __init__(self): pass @huey.periodic_task(crontab(minute='*', hour='*')) def cleanUserDatabase(): '''Huey task to routinely clean the database of all user accounts that are unverified ''' # Set to Five Minutes for debug conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back) # expired = time.time() - 86400 # One day ago expired = time.time() - 300 # Five minute(s) ago r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up result = r.table(c.user_table).filter((r.row['created'] < expired) and (r.row['verified'] == False))['id'].coerce_to('array').run(conn) # The above string asks the database server to locate all items out of the user table where 'created' is older than 'expired' and verified is false if len(result) > 0: database.delete(c.user_table, conn, database.args(result)) # Delete all the IDs recieved by the query string out of the 'user' database database.delete(c.auth_table, conn, database.args(result)) # Delete all the IDs recieved by the query string out of the 'auth' database print "Removed {} unverified users from the database".format(len(result)) conn.close() # Clean up the database connection @huey.periodic_task(crontab(minute='*', hour='*')) def cleanLoginSessions(): '''Huey task to clear login sessions out of the database''' conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back) expired = time.time() - 120 # Two minute(s) ago r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up result = r.table(c.auth_table).update(lambda row:{'login_sessions':row['login_sessions'].filter(lambda session:session['timestamp'] > expired)}).run(conn) # The above string asks the database server to find all login sessions in the table 'auth' with a timestamp older than 'expired' and delete them # This is a heavy action - should be stress tested if result['replaced'] > 0: print "Removed expired login_sessions for {} users from the database".format(result['replaced']) conn.close() # Clean up the database connection
def test_crontab_hour(self): # validates the following hours valids = [0, 1, 4, 6, 8, 9, 12, 18] validate_h = crontab(hour='8-9,*/6,1,4') for x in range(24): res = validate_h(datetime.datetime(2011, 1, 1, x)) self.assertEqual(res, x in valids) edge = crontab(hour=0) self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0))) self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0)))
def set_index_crontab(value: str, conn: Connection) -> None: """ :raises InvalidConfig: If the crontab is syntactically invalid. """ try: crontab(**parse_crontab(value), strict=True) except ValueError: raise InvalidConfig(f"{value} is not a valid crontab.") conn.execute( "UPDATE system__config SET value = ? WHERE key = ?", (value, INDEX_CRONTAB), )
def test_crontab_minute(self): # validates the following minutes valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54] validate_m = crontab(minute='4,8-9,*/6,1') for x in range(60): res = validate_m(datetime.datetime(2011, 1, 1, 1, x)) self.assertEqual(res, x in valids) # We don't ensure *every* X minutes, but just on the given intervals. valids = [0, 16, 32, 48] validate_m = crontab(minute='*/16') for x in range(60): res = validate_m(datetime.datetime(2011, 1, 1, 1, x)) self.assertEqual(res, x in valids)
def test_crontab_all_together(self): # jan 1, 2011 is a saturday # may 1, 2011 is a sunday validate = crontab(month='1,5', day='1,4,7', day_of_week='0,6', hour='*/4', minute='1-5,10-15,50') self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11))) self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50))) self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1))) # fails validation on month self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11))) # fails validation on day self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11))) # fails validation on day_of_week self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11))) # fails validation on hour self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11))) # fails validation on minute self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))
def test_crontab_all_together(self): # jan 1, 2011 is a saturday # may 1, 2011 is a sunday validate = crontab( month='1,5', day='1,4,7', day_of_week='0,6', hour='*/4', minute='1-5,10-15,50' ) self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11))) self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50))) self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1))) # fails validation on month self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11))) # fails validation on day self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11))) # fails validation on day_of_week self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11))) # fails validation on hour self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11))) # fails validation on minute self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))
def test_crontab_month(self): # validates the following months, 1, 4, 7, 8, 9 valids = [1, 4, 7, 8, 9] validate_m = crontab(month='1,4,*/6,8-9') for x in range(1, 13): res = validate_m(datetime.datetime(2011, x, 1)) self.assertEqual(res, x in valids)
def test_task_decorators(self): huey = RedisHuey() def test_fn(): pass test_fn_task = huey.task()(test_fn) test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn) self.assertTrue(isinstance(test_fn_task, TaskWrapper)) self.assertTrue(test_fn_task.func is test_fn) self.assertTrue(isinstance(test_fn_cron, TaskWrapper)) self.assertTrue(test_fn_cron.func is test_fn) test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task) self.assertTrue(isinstance(test_cron_task, TaskWrapper)) self.assertTrue(test_cron_task.func is test_fn)
def test_crontab_day(self): # validates the following days valids = [1, 4, 7, 8, 9, 13, 19, 25, 31] validate_d = crontab(day='*/6,1,4,8-9') for x in range(1, 32): res = validate_d(datetime.datetime(2011, 1, x)) self.assertEqual(res, x in valids)
def schedule_subscriptions_check(user, minutes='0', hours='12'): def wrapper(): subscriptions_check(user) schedule = crontab(minute=minutes, hour=hours) task_name = f'user_{user.id}_subscriptions' db_periodic_task(schedule, name=task_name)(wrapper)
def test_crontab_minute(self): # validates the following minutes valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54] validate_m = crontab(minute='4,8-9,*/6,1') for x in range(60): res = validate_m(datetime.datetime(2011, 1, 1, 1, x)) self.assertEqual(res, x in valids)
def test_crontab_day_of_week(self): # validates the following days of week # jan, 1, 2011 is a saturday valids = [2, 4, 9, 11, 16, 18, 23, 25, 30] validate_dow = crontab(day_of_week='0,2') for x in range(1, 32): res = validate_dow(datetime.datetime(2011, 1, x)) self.assertEqual(res, x in valids)
def test_crontab_sunday(self): for dow in ('0', '7'): validate = crontab(day_of_week=dow, hour='0', minute='0') valid = set((2, 9, 16, 23, 30)) for x in range(1, 32): if x in valid: self.assertTrue(validate(datetime.datetime(2011, 1, x))) else: self.assertFalse(validate(datetime.datetime(2011, 1, x)))
def schedule(action): cfg = app.config.get("HUEY_SCHEDULE", {}) action_cfg = cfg.get(action) if action_cfg is None: raise RuntimeError( "No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again." .format(x=action)) return crontab(**action_cfg)
def test_crontab_day(self): # validates the following days valids = [1, 4, 7, 8, 9, 13, 19, 25, 31] validate_d = crontab(day='*/6,1,4,8-9') for x in range(1, 32): res = validate_d(datetime.datetime(2011, 1, x)) self.assertEqual(res, x in valids) valids = [1, 11, 21, 31] validate_d = crontab(day='*/10') for x in range(1, 32): res = validate_d(datetime.datetime(2011, 1, x)) self.assertEqual(res, x in valids) valids.pop() # Remove 31, as feb only has 28 days. for x in range(1, 29): res = validate_d(datetime.datetime(2011, 2, x)) self.assertEqual(res, x in valids)
def schedule_alarm(alarm_name, cron_minutes, cron_hours, cron_days): def wrapper(): alarm(alarm_name) # The schedule that was specified for this task. schedule = crontab(minute=cron_minutes, hour=cron_hours, day_of_week=cron_days) periodic_task(schedule, name=alarm_name)(wrapper)
def load_periodic_tasks(self): """ This method read schedule.info file from the instance. At the same folder level that manage.py file. :return: """ schedule_process = self.execute_command('makeschedule') while self.is_running(schedule_process): time.sleep(0.5) if os.path.isfile(os.path.join(os.path.dirname(self.script_path), 'schedule.info')): self._logger.debug('Schedule info created') else: self._logger.debug('Schedule info not found') return info_file = os.path.join(os.path.dirname(self.script_path), 'schedule.info') with open(info_file, 'r') as f: lines = f.readlines() for ln in lines: ln = ln.rstrip().strip().rstrip() if ln.startswith(';') or ln.startswith('#'): continue info = ln.split() if len(info) == 6: self._logger.info('Added periodic method: %s', ln) self.periodic_tasks.append({ 'method': info[5], 'validate_datetime': crontab( minute=info[0], hour=info[1], day_of_week=info[2], day=info[3], month=info[4]) }) elif len(info) > 1: self._logger.info('Invalid cron line.') if len(self.periodic_tasks) == 0: self._logger.info('No periodic task found')
from huey import crontab from huey.contrib.djhuey import db_periodic_task from influxdb_metrics.loader import write_points from foodsaving.users.stats import get_users_stats from foodsaving.utils import stats_utils @db_periodic_task(crontab(hour='*/6', minute=5)) # every 6 hours def record_user_stats(): stats_utils.periodic_task('users__record_user_stats') fields = get_users_stats() write_points([{ 'measurement': 'karrot.users', 'fields': fields, }])
import time import redis from django.conf import settings from huey import crontab from huey.contrib.djhuey import db_periodic_task from .models import Log from .main import post_blood_weather, update_blood_groups """ Method to fetch blood groups and update status """ def fetch_and_update(): blood_groups, log = update_blood_groups() post_blood_weather(blood_groups, log) """ Periodic task to update blood statuses """ @db_periodic_task(crontab(minute='*/15')) def main_blood_groups_task(): fetch_and_update() time.sleep(settings.BLOOD_FETCH_INTERVAL) return True
multiprocessing.set_start_method("fork") except RuntimeError: # sometimes it has already been set by # other libs pass redis_host = None if settings.cache_url: redis_host = settings.cache_url.host huey = PriorityRedisHuey("opennem.scheduler", host=redis_host) # export tasks @huey.periodic_task(crontab(minute="*/5"), priority=90) @huey.lock_task("schedule_live_tasks") def schedule_live_tasks() -> None: if settings.workers_run: export_power(priority=PriorityType.live) export_flows() @huey.periodic_task(crontab(minute="*/30"), priority=90) @huey.lock_task("schedule_custom_tasks") def schedule_custom_tasks() -> None: if settings.workers_run: export_electricitymap() @huey.periodic_task(crontab(hour="13"), priority=50)
from huey import RedisHuey, crontab from scrape import get_data_send_mail queue = RedisHuey("scrape-trending") @queue.periodic_task(crontab(minute="0", hour="*")) def send_mail(): get_data_send_mail()
from huey import crontab from huey.contrib.djhuey import db_periodic_task import logging from danceschool.core.constants import getConstant from .helpers import createExpenseItemsForEvents, createExpenseItemsForVenueRental,createRevenueItemsForRegistrations # Define logger for this file logger = logging.getLogger(__name__) @db_periodic_task(crontab(minute='*/60')) def updateFinancialItems(): ''' Every hour, create any necessary revenue items and expense items for activities that need them. ''' logger.info('Creating automatically-generated financial items.') if getConstant('financial__autoGenerateExpensesEventStaff'): createExpenseItemsForEvents() if getConstant('financial__autoGenerateExpensesVenueRental'): createExpenseItemsForVenueRental() if getConstant('financial__autoGenerateRevenueRegistrations'): createRevenueItemsForRegistrations()
def retry_command(k, always_fail=True): if k not in state: if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @queue_command(test_invoker, retries=3, retry_delay=10) def retry_command_slow(k, always_fail=True): if k not in state: if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @periodic_command(test_invoker, crontab(minute='0')) def every_hour(): state['p'] = 'y' # create a log handler that will track messages generated by the consumer class TestLogHandler(logging.Handler): def __init__(self, *args, **kwargs): self.messages = [] logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.messages.append(record.getMessage()) class SkewConsumerTestCase(unittest.TestCase): def setUp(self):
def _throw_error_task(message=None): raise TestException(message or 'bampf') throw_error_task = huey.task()(_throw_error_task) throw_error_task_res = huey_results.task()(_throw_error_task) @huey_results.task() def add_values(a, b): return a + b @huey_results.task() def add_values2(a, b): return a + b @huey_results.periodic_task(crontab(minute='0')) def hourly_task2(): state['periodic'] = 2 @huey_results.task() def returns_none(): return None @huey_store_none.task() def returns_none2(): return None class BaseQueueTestCase(BaseTestCase): def setUp(self): global state
import sqlite3 from conreq.utils.generic import get_database_type from django.conf import settings from django.db import connection from huey import crontab from huey.contrib.djhuey import db_periodic_task DB_ENGINE = get_database_type() HUEY_FILENAME = getattr(settings, "HUEY_FILENAME") @db_periodic_task(crontab(minute="0", hour="12", day="1/1")) def bg_tasks_vacuum(): """Periodically preforms a SQLITE vacuum on the background task database.""" with sqlite3.connect(HUEY_FILENAME) as cursor: cursor.execute("VACUUM") @db_periodic_task(crontab(minute="0", hour="12", day="1/1")) def db_vacuum(): """Periodically performs any cleanup tasks needed for the default database.""" if DB_ENGINE == "SQLITE3": with connection.cursor() as cursor: cursor.execute("VACUUM")
class TestException(Exception): pass @huey.task() def _throw_error_task(message=None): raise TestException(message or 'bampf') throw_error_task = huey.task()(_throw_error_task) throw_error_task_res = huey_results.task()(_throw_error_task) @huey_results.task() def add_values(a, b): return a + b @huey_results.periodic_task(crontab(minute='0')) def hourly_task2(): state['periodic'] = 2 @huey_results.task() def returns_none(): return None @huey_store_none.task() def returns_none2(): return None class BaseQueueTestCase(BaseTestCase): def setUp(self): global state
#!/usr/bin/env python # -*- coding: utf-8 -*- # Survaider from huey import crontab from jupiter import huey from jupiter.sentient.model import AspectQ import datetime @huey.periodic_task(crontab(minute='*/1')) def spawn_processor_m_30(): """These processors are run every 30 minutes Processors Index ---------------- These processors are explicitly run: - External API Calls - (Extend this list as needed) Developer Notes --------------- All the tasks (such as scraping, or, machine learning etc.) must be *spawned* from here. It is important to note that these tasks are **required** to be non-blocking and must be huey tasks. To prevent any magic to happen, do **not** spawn any periodic tasks inside. """ print ("Process Running-- delay : 2min") # print(len(AspectQ.objects)) for obj in AspectQ.objects:
def retry_command(k, always_fail=True): if k not in state: if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @test_huey.task(retries=3, retry_delay=10) def retry_command_slow(k, always_fail=True): if k not in state: if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @test_huey.periodic_task(crontab(minute='0')) def every_hour(): state['p'] = 'y' # Create a log handler that will track messages generated by the consumer. class TestLogHandler(logging.Handler): def __init__(self, *args, **kwargs): self.messages = [] logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.messages.append(record.getMessage()) class ConsumerTestCase(unittest.TestCase):
return a * b @db_task() # Opens DB connection for duration of task. def slow(n): tprint('going to sleep for %s seconds' % n) time.sleep(n) tprint('finished sleeping for %s seconds' % n) return n @task(retries=1, retry_delay=5, context=True) def flaky_task(task=None): if task is not None and task.retries == 0: tprint('flaky task succeeded on retry.') return 'succeeded on retry.' tprint('flaky task is about to raise an exception.', 31) raise Exception('flaky task failed!') # Periodic tasks. @periodic_task(crontab(minute='*/2')) def every_other_minute(): tprint('This task runs every 2 minutes.', 35) @periodic_task(crontab(minute='*/5')) def every_five_mins(): tprint('This task runs every 5 minutes.', 34)
send_mail(subject, body, from_, tos) def _get_comment_body(blogitem, blogcomment): base_url = "https://%s" % Site.objects.get_current().domain if "peterbecom.local" in base_url: base_url = "http://*****:*****@periodic_task(crontab(hour="*")) def run_populate_blogitem_daily_hits(): date = timezone.now() - datetime.timedelta(days=1) try: sum_count, items_count = BlogItemDailyHits.rollup_date(date) print( "Rolled up {:,} blogitems a total of {:,} hits".format( items_count, sum_count ) ) except BlogItemDailyHitsExistingError: print("Already rolled up for {}".format(date))
@db_task() # Opens DB connection for duration of task. def slow(n): tprint('going to sleep for %s seconds' % n) time.sleep(n) tprint('finished sleeping for %s seconds' % n) return n @task(retries=1, retry_delay=5, context=True) def flaky_task(task=None): if task is not None and task.retries == 0: tprint('flaky task succeeded on retry.') return 'succeeded on retry.' tprint('flaky task is about to raise an exception.', 31) raise Exception('flaky task failed!') # Periodic tasks. @periodic_task(crontab(minute='*/2')) def every_other_minute(): tprint('This task runs every 2 minutes.', 35) @periodic_task(crontab(minute='*/5')) def every_five_mins(): tprint('This task runs every 5 minutes.', 34)
from django.conf import settings from django.db.models import F, OuterRef, Subquery from django.utils import timezone from django.utils.dateparse import parse_datetime from huey import crontab from huey.contrib.djhuey import db_task, db_periodic_task, HUEY from huey.exceptions import TaskLockedException from camp.apps.monitors.models import Entry from camp.apps.monitors.purpleair import api from camp.apps.monitors.purpleair.models import PurpleAir @db_periodic_task(crontab(minute='*/2'), priority=50) def import_recent_data(): print('[import_recent_data]') if HUEY.pending_count() > settings.MAX_QUEUE_SIZE: return for monitor in PurpleAir.objects.all(): print('\n' * 10, '[import_recent_data]', monitor.name, '\n' * 10) import_monitor_data.schedule([monitor.pk], delay=1, priority=30) @db_task() def import_monitor_history(monitor_id, end=None): monitor = PurpleAir.objects.get(pk=monitor_id) end = end or timezone.now() feeds = monitor.get_feeds(end=end, results=1000)
res_queue = DummyQueue(res_queue_name) res_store = DummyDataStore(res_queue_name) res_huey = Huey(res_queue, res_store, schedule) res_huey_nones = Huey(res_queue, res_store, store_none=True) # store some global state state = {} # create a decorated queue command @huey.task() def add(key, value): state[key] = value # create a periodic queue command @huey.periodic_task(crontab(minute='0')) def add_on_the_hour(): state['periodic'] = 'x' # define a command using the class class AddTask(QueueTask): def execute(self): k, v = self.data state[k] = v # create a command that raises an exception class BampfException(Exception): pass @huey.task() def throw_error():
@huey.task() def send_mail(content): smtp_config = setting_col.find_one({'key': 'mail'}) receivers = [data.get('mail') for data in notice_col.find({})] try: if mail_notice(smtp_config, receivers, content): logger.info('邮件发送成功') else: logger.critical('Error: 无法发送邮件') except smtplib.SMTPException as error: logger.critical('Error: 无法发送邮件 {}'.format(error)) @huey.periodic_task(crontab(minute='*/2')) def update_rate_remain(): for account in github_col.find(): github_username = account.get('username') github_password = account.get('password') try: g = Github(github_username) github_col.update_one({'username': github_username}, { '$set': { 'rate_remaining': int(g.get_rate_limit().search.remaining), 'rate_limit': int(g.get_rate_limit().search.limit) } }) except Exception as error: logger.error(error)
def explicit_retry(k): if k not in state: state[k] = 'fixed' raise RetryTask() return state[k] @test_huey.task(retries=1, include_task=True) def retry_with_task(a, b, task=None): assert task is not None if a + b < 0: raise RetryTask() return a + b @test_huey.periodic_task(crontab(minute='2')) def hourly_task(): state['p'] = 'y' @test_huey.periodic_task(crontab(minute='3'), retries=3) def hourly_task2(): try: state['p2'] += 1 except KeyError: state['p2'] = 1 raise @test_huey.task(retries=2) @test_huey.lock_task('test-lock')
@test_huey.task(retries=2) def explicit_retry(k): if k not in state: state[k] = 'fixed' raise RetryTask() return state[k] @test_huey.task(retries=1, include_task=True) def retry_with_task(a, b, task=None): assert task is not None if a + b < 0: raise RetryTask() return a + b @test_huey.periodic_task(crontab(minute='2')) def hourly_task(): state['p'] = 'y' @test_huey.periodic_task(crontab(minute='3'), retries=3) def hourly_task2(): try: state['p2'] += 1 except KeyError: state['p2'] = 1 raise @test_huey.task(retries=2) @test_huey.lock_task('test-lock') def locked_task(a, b): return a + b
if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @test_huey.task(retries=3, retry_delay=10) def retry_task_delay(k, always_fail=True): if k not in state: if not always_fail: state[k] = 'fixed' raise Exception('fappsk') return state[k] @test_huey.periodic_task(crontab(minute='2')) def hourly_task(): state['p'] = 'y' class TestExecution(HueyTestCase): def create_consumer(self, worker_type='thread'): return Consumer(self.huey, max_delay=0.1, workers=2, worker_type=worker_type) def test_threaded_execution(self): consumer = self.create_consumer() r1 = modify_state('k1', 'v1') r2 = modify_state('k2', 'v2')
import random import time from huey import crontab from config import huey @huey.task() def count_beans(num): print('-- counted %s beans --' % num) return 'Counted %s beans' % num @huey.periodic_task(crontab(minute='*/5')) def every_five_mins(): print('Consumer prints this every 5 mins') @huey.task(retries=3, retry_delay=10) def try_thrice(): if random.randint(1, 3) == 1: print('OK') else: print('About to fail, will retry in 10 seconds') raise Exception('Crap something went wrong') @huey.task() def slow(n): time.sleep(n) print('slept %s' % n)
# from app.tasks.gift_code_management import send_gift_code_winner_approval_pm from cubersio.tasks.reddit import prepare_new_competition_notification,\ prepare_end_of_competition_info_notifications from . import huey # ------------------------------------------------------------------------------------------------- if app.config['IS_DEVO']: # don't run as periodic in devo WRAP_WEEKLY_COMP_SCHEDULE = lambda _ : False RUN_RANKINGS_SCHEDULE = lambda _ : False else: # Run the rankings task several hours after the comp stuff. I think we're having memory issues. WRAP_WEEKLY_COMP_SCHEDULE = crontab(day_of_week='1', hour='2', minute='0') RUN_RANKINGS_SCHEDULE = crontab(day_of_week='1', hour='6', minute='0') # ------------------------------------------------------------------------------------------------- @huey.periodic_task(RUN_RANKINGS_SCHEDULE) def run_weekly_site_rankings(): """ A periodic task to run the site rankings stuff weekly. """ run_user_site_rankings() @huey.task() def run_user_site_rankings(): """ A task to run the calculations to update user site rankings based on the latest data. """