def __init__(self, options): BaseQueue.__init__(self, options) # TODO: We should rethink ``store_errors=False`` -- how do we notice errors? self._huey = RedisHuey(result_store=False, store_none=False, store_errors=False, **options) self._jobs = {}
def test_non_global_task_registry(self): huey = RedisHuey(global_registry=False) @huey.task() def test(): return 'test' self.assertIn('queuecmd_test', huey.registry._registry) huey2 = RedisHuey(global_registry=False) self.assertNotIn('queuecmd_test', huey2.registry._registry)
class HueyQueue(BaseQueue): def __init__(self, options): BaseQueue.__init__(self, options) # TODO: We should rethink ``store_errors=False`` -- how do we notice errors? self._huey = RedisHuey(result_store=False, store_none=False, store_errors=False, **options) self._jobs = {} @property def huey(self): return self._huey @property def jobs(self): return self._jobs def register_job(self, name, func): if name in self._jobs: raise QueueError(u"Job function {!r} already exists".format(name)) self._jobs[name] = self._huey.task(name=name)(func) def enqueue(self, name, args, kwargs): if name not in self._jobs: raise QueueError(u"Unknown job: {!r}".format(name)) log.info(u"Sending {!r} job to the queue ...".format(name)) # We do not care about results self._jobs[name](*args, **kwargs)
def init_huey(settings: dict) -> RedisHuey: global huey huey = RedisHuey( url=settings["zam.tasks.redis_url"], always_eager=asbool(settings.get("zam.tasks.always_eager", "False")), ) return huey
def test_task_decorators(self): huey = RedisHuey() def test_fn(): pass test_fn_task = huey.task()(test_fn) test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn) self.assertTrue(isinstance(test_fn_task, TaskWrapper)) self.assertTrue(test_fn_task.func is test_fn) self.assertTrue(isinstance(test_fn_cron, TaskWrapper)) self.assertTrue(test_fn_cron.func is test_fn) test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task) self.assertTrue(isinstance(test_cron_task, TaskWrapper)) self.assertTrue(test_cron_task.func is test_fn)
def create_huey(): """ Huey is our asynchronous task scheduler. https://huey.readthedocs.io/en/latest/ """ if settings.PROCESS_ASYNC_TASKS: return RedisHuey( connection_pool=cache.RedisCache.connect().connection_pool) return DummyHuey()
def __init__(self): self.logger = logging.getLogger('App.App') self.logger.setLevel(logging.DEBUG) self.config = config self.db = DB(config) self.huey = RedisHuey(host="redis", result_store=False) self.web3 = Web3(HTTPProvider(config.HTTP_PROVIDER_URL)) self._tokens = None self.updateTokens()
def __init__(self, name, connection_pool): self.name = name self.clean_name = self.clean_queue_name(self.name) self.huey = RedisHuey(self.name, connection_pool=connection_pool) self.event_handlers = { EVENT_FINISHED: self.event_finished, EVENT_ENQUEUED: self.event_enqueued, EVENT_STARTED: self.event_started, EVENT_ERROR_TASK: self.event_error, }
def _get_asynchronous_result(huey_app: RedisHuey, huey_task_id: str): try: huey_task = huey_app.result(huey_task_id) except TaskException as e: try: for import_exception in imported_exceptions: exec(import_exception) task_exception = eval(e.metadata["error"]) except NameError: task_exception = Exception(e.metadata["error"]) raise task_exception return huey_task
def connect_huey(): global huey try: queue_name = get_settings().HUEY_TASK_QUEUES url = get_settings().HUEY_REDIS_CONNECTION if url != "" and queue_name != "": huey = RedisHuey(queue_name, url=url, blocking=False) print("Huey Connect") else: print("Huey No Connection ") except Exception as e: print("Huey Connect Fail: " + str(e)) return huey
def build_async_application(config: dict, **kwargs) -> RedisHuey: """ This function should be called within a python module called 'asynchronous_server.py' Build a huey application with given configuration and modules :param config: Dictionary with following structure: { 'asynchronous': { 'broker': ..., } } :param kwargs: Additional Huey arguments :return: RedisHuey Application """ logger.info(f"Starting Huey server") return RedisHuey( os.getenv("CONTAINER_NAME", "LOCAL"), url=config["asynchronous"]["broker"], **kwargs, )
def create_huey(config): _storage_args = { 'read_timeout': config.get('HUEY_STORAGE_READ_TIMEOUT', 1), 'max_errors': config.get('HUEY_STORAGE_MAX_ERRORS', 1000), 'connection_pool': config.get('HUEY_STORAGE_CONNECTION_POOL', None), 'url': config.get('HUEY_STORAGE_URL', None), } _huey = RedisHuey(name=config.get('HUEY_TASK_QUEUE_NAME', 'FLASK-HUEY-EXAMPLE'), result_store=config.get('HUEY_RESULT_STORE', False), events=config.get('HUEY_EVENTS', True), store_none=config.get('HUEY_STORE_NONE', False), always_eager=config.get('HUEY_ALWAYS_EAGER', False), store_errors=config.get('HUEY_STORE_ERRORS', True), blocking=config.get('HUEY_BLOCKING', False), **_storage_args) return _huey
import smtplib import datetime from huey import RedisHuey, crontab from . import database, handler, crypto iterate = handler.Iterate() webdata = handler.WebData() quicrypt = crypto.QuiCrypt() c = handler.Bunch(webdata.loader('config.json')) try: huey = RedisHuey('hub-queue', host=c.db_host, port=6379) except: huey = RedisHuey('hub-queue', host=c.db_back, port=6379) class Automation(object): def __init__(self): pass @huey.periodic_task(crontab(minute='*', hour='*')) def cleanUserDatabase(): '''Huey task to routinely clean the database of all user accounts that are unverified ''' # Set to Five Minutes for debug conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back) # expired = time.time() - 86400 # One day ago expired = time.time() - 300 # Five minute(s) ago r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up
import zhihu from pymongo import MongoClient from huey import RedisHuey from requests.adapters import HTTPAdapter from zhihu import ANONYMOUS from common import * if hasattr(os, '_called_from_test'): from client_pool import get_client_test as get_client # don't use proxy else: from client_pool import get_client_test as get_client # use kunpeng proxy from utils import config_smtp_handler huey = RedisHuey() logger = logging.getLogger('huey.consumer') logger2 = logging.getLogger('huey.consumer.Worker') logger2.setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) if os.path.isfile(logging_config_file): with open(logging_config_file, 'rt') as f: config = json.load(f) mail_handler_cfg = config['handlers']['mail_handler'] smtp_handler = logging.handlers.SMTPHandler( mailhost=(mail_handler_cfg['mailhost'], 25), fromaddr=mail_handler_cfg['fromaddr'], toaddrs=mail_handler_cfg['toaddrs'], subject="Huey Error")
import pickle from huey import crontab from huey import exceptions as huey_exceptions from huey import RedisHuey from huey.api import Huey from huey.api import QueueTask from huey.api import TaskWrapper from huey.constants import EmptyData from huey.registry import registry from huey.storage import RedisStorage from huey.tests.base import b from huey.tests.base import BaseTestCase from huey.utils import local_to_utc huey = RedisHuey(result_store=False, events=False, blocking=False) huey_results = RedisHuey(blocking=False, max_errors=10) huey_store_none = RedisHuey(store_none=True, blocking=False) # Global state. state = {} @huey.task() def put_data(key, value): state[key] = value @huey.task(include_task=True) def put_data_ctx(key, value, task=None): state['last_task_class'] = type(task).__name__ @huey_results.task(include_task=True)
import time from huey import RedisHuey huey = RedisHuey("predictions", host="localhost", port=16379) @huey.task() def predict(features): time.sleep(5) s = sum(n for n in features.values()) print("Predicted:", s) return s def create_prediction(features): return predict(features)
from huey import RedisHuey, crontab from portality.core import app main_queue = RedisHuey('doaj_main_queue', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False)) long_running = RedisHuey('doaj_long_running', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False)) def schedule(action): cfg = app.config.get("HUEY_SCHEDULE", {}) action_cfg = cfg.get(action) if action_cfg is None: raise RuntimeError( "No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again." .format(x=action)) return crontab(**action_cfg) def configure(action): cfg = app.config.get("HUEY_TASKS", {}) action_cfg = cfg.get(action) if action_cfg is None: raise RuntimeError( "No task configuration for action '{x}'. Define this in HUEY_TASKS first then try again."
import pandas as pd from sklearn.preprocessing import LabelEncoder import pystan import pickle from huey import RedisHuey huey = RedisHuey('iris') def get_iris_dataset(): iris_file = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" return pd.read_csv(iris_file, names=[ 'sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species' ]) def format_stan_data(iris_data): # Simple multi-level model, predict petal length using sepal length # and random effects from each species. species = LabelEncoder() iris_data['species_num'] = species.fit_transform(iris_data['species']) + 1 stan_iris_data = { 'n': iris_data.shape[0], 'n_species': max(iris_data['species_num']), 'sepal_length': iris_data['sepal_length'], 'species': iris_data['species_num'], 'petal_length': iris_data['petal_length']
import os import json import requests import random from huey import RedisHuey # worker huey = RedisHuey(url=os.getenv('REDIS_URL')) # task @huey.task(retries=5, retry_delay=5) def get_random_num(): print("This is a task to get a random number") num = random.randint(1, 3) print("Random number is {}".format(num)) if num == 1: # if number is 1, the task was successful return True else: # if number is not 1, raise an error (so that we see how a failed task looks like and what happens next) raise Exception("Error in the worker... :(") # task @huey.task(retries=10, retry_delay=600) def send_email_task(receiver_email, subject, text): sender_email = os.getenv("MY_SENDER_EMAIL") # Your website's official email address api_key = os.getenv('SENDGRID_API_KEY') if sender_email and api_key: url = "https://api.sendgrid.com/v3/mail/send"
TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) STATIC_ROOT = os.path.join(PROJECT_DIR, 'static') from huey import RedisHuey from redis import ConnectionPool import os huey_app = os.getenv('huey_app', 'production_app') always_eager = False if huey_app != 'production_app': always_eager = False print('Always eager huey!') pool = ConnectionPool.from_url( 'redis://*****:*****@ec2-34-252-182-25.eu-west-1.compute.amazonaws.com:27659' ) HUEY = RedisHuey(huey_app, connection_pool=pool, always_eager=always_eager) print('huey app:', HUEY.name)
return load_class(path + '.Components') except ImportError: config_error('Unable to import %s: "%s"' % (key, path)) try: HUEY = getattr(settings, 'HUEY', None) except: config_error('Error encountered reading settings.HUEY') if HUEY is None: try: from huey import RedisHuey except ImportError: config_error('Error: Huey could not import the redis backend. ' 'Install `redis-py`.') HUEY = RedisHuey(default_queue_name()) if not isinstance(HUEY, Huey): Queue, DataStore, Schedule = dynamic_import(HUEY, 'backend') name = HUEY.get('name') or default_queue_name() conn = HUEY.get('connection', {}) always_eager = HUEY.get('always_eager', False) HUEY = Huey( Queue(name, **conn), DataStore(name, **conn), Schedule(name, **conn), always_eager=always_eager) task = HUEY.task periodic_task = HUEY.periodic_task
import email from os import getenv from huey import RedisHuey import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.header import Header huey = RedisHuey(utc=False, host=getenv('REDIS_HOST') or '127.0.0.1', port=int(getenv('REDIS_PORT') or 6379), db=int(getenv('REDIS_DB') or 0)) @huey.task() # 发送邮件案例 def send_email(content, name, phone, e_mail): """发送邮件""" try: username = '******' # 发送人邮箱 password = '******' # 发送密钥 replyto = '' # 回复邮件接受人邮箱 receiver = '*****@*****.**' # 接受者邮箱 msg = MIMEMultipart('html') msg['Subject'] = Header('邮件标题') from_header = Header('*****@*****.**', 'utf-8') # 来自... from_header.append('<{}>'.format(username), 'ascii') msg['From'] = from_header msg['To'] = receiver msg['Reply-to'] = replyto msg['Message-id'] = email.utils.make_msgid()
def get_storage(self, **kwargs): return BaseStorage() class BrokenRedisStorage(RedisStorage): def dequeue(self): raise ValueError('broken redis dequeue') broken_redis_storage = BrokenRedisStorage() class BrokenHuey(Huey): def get_storage(self): return broken_redis_storage dummy_huey = DummyHuey() test_huey = RedisHuey('testing', blocking=False, read_timeout=0.1) # Logger used by the consumer. logger = logging.getLogger('huey.consumer') logger.addHandler(logging.NullHandler()) # Create a log handler that will track messages generated by the consumer. class CaptureLogs(logging.Handler): def __init__(self, *args, **kwargs): self.messages = [] logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.messages.append(record.getMessage()) def __enter__(self):
from concurrent.futures import ProcessPoolExecutor, as_completed from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer import requests import botometer from settings import Settings from elasticsearch import Elasticsearch from huey import RedisHuey import re import sys STEP = 100 TWITTER_DATETIME_PATTERN = "ddd MMM DD HH:mm:SS Z YYYY" FIXED_TWITTER_DATE_TIME = re.compile('\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z') MAX_DELTA_BOTOMETER = 3600 * 24 * 2 # 2 Days huey = RedisHuey('esK3K2') def esK3K2_ascii_art(): print(" __ _______ __ _____ ") print(" ___ _____/ //_/__ // //_/__ \ ") print(" / _ \/ ___/ ,< /_ </ ,< __/ / ") print("/ __(__ ) /| |___/ / /| |/ __/ ") print("\___/____/_/ |_/____/_/ |_/____/ ") print(" ") def dotter(d: dict, key, dots): """Creates a list of key of a dict is very useful to use with Scalpl Arguments: d {dict} -- Input dictionary key {[type]} -- Recursive keys
import os from huey import RedisHuey huey_queue = RedisHuey("licenta", host="localhost") ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) DB_PATH = os.path.join(ROOT_DIR, 'db.json')
from huey import RedisHuey huey = RedisHuey('distributor', host='localhost')
from huey import RedisHuey huey = RedisHuey()
import pickle import requests from bs4 import BeautifulSoup from huey import RedisHuey, crontab from pushbullet import Pushbullet from redis import StrictRedis, BlockingConnectionPool url = os.getenv('URL', 'https://shop.lego.com/en-CH/Millennium-Falcon-75192') redis_host = os.getenv('REDIS_HOST', 'localhost') redis_port = os.getenv('REDIS_PORT', 6379) redis_db = os.getenv('REDIS_DB', 0) pushbullet_api_key = os.getenv('PUSHBULLET_API_KEY', '') pool = BlockingConnectionPool(host=redis_host, port=redis_port, db=redis_db) redis = StrictRedis(connection_pool=pool) huey = RedisHuey('millennium-falcon-checker', connection_pool=pool) pb = Pushbullet(pushbullet_api_key) @huey.periodic_task(crontab(minute='*/30')) def check_status_task(): check_status() def check_status(): millennium_falcon_response = requests.get(url) try: millennium_falcon_response.raise_for_status() soup = BeautifulSoup(millennium_falcon_response.content, 'html.parser') if is_available(soup): print(' Millennium Falcon is available')
from huey import RedisHuey, crontab from scrape import get_data_send_mail queue = RedisHuey("scrape-trending") @queue.periodic_task(crontab(minute="0", hour="*")) def send_mail(): get_data_send_mail()
import datetime from huey import RedisHuey, crontab from db.models import init_db, dbsession from db.models import Source from common.scrapers import flipp init_db() huey = RedisHuey('ffd_cron', host='redis', immediate_use_memory=True) @huey.periodic_task(crontab(minute='*')) def pull_sources(): yesterday = datetime.date.today() - datetime.timedelta(days=7) source = Source.find_by_method('flipp').filter( Source.last_updated < yesterday).first() if source: meta = source.meta.copy() meta['source_id'] = source.id meta['postal_code'] = source.store.postal_code meta['chain'] = source.store.chain.name flipp.scrape(meta)
import random import requests import smtplib import time from ipaddress import ip_address import tldextract from github import Github from huey import RedisHuey, crontab from pymongo import errors, DESCENDING, ASCENDING from config.database import result_col, query_col, blacklist_col, notice_col, github_col, setting_col, REDIS_HOST, \ REDIS_PORT from utils.date import timestamp from utils.log import logger from utils.notice import mail_notice huey = RedisHuey('hawkeye', host=REDIS_HOST, port=int(REDIS_PORT)) base_path = os.path.split(os.path.realpath(__file__))[0] extract = tldextract.TLDExtract(cache_file='{}/.tld_set'.format(base_path)) if setting_col.count({ 'key': 'task', 'minute': { '$exists': True }, 'page': { '$exists': True } }): minute = int(setting_col.find_one({'key': 'task'}).get('minute')) setting_col.update_one( {'key': 'task'},
from huey import RedisHuey, crontab # from opennem.api.exporter import wem_run_all from opennem.settings import settings huey = RedisHuey("opennem.exporter", host=settings.cache_url.host) # @huey.periodic_task(crontab(minute="*/10")) # def wem_export_task(): # wem_run_all()
import datetime import pickle from huey import crontab from huey import exceptions as huey_exceptions from huey import RedisHuey from huey.api import Huey from huey.api import QueueTask from huey.registry import registry from huey.storage import RedisStorage from huey.tests.base import b from huey.tests.base import BaseTestCase from huey.utils import EmptyData from huey.utils import local_to_utc huey = RedisHuey(result_store=False, events=False, blocking=False) huey_results = RedisHuey(blocking=False, max_errors=10) huey_store_none = RedisHuey(store_none=True, blocking=False) # Global state. state = {} @huey.task() def put_data(key, value): state[key] = value @huey.task(include_task=True) def put_data_ctx(key, value, task=None): state['last_task_class'] = type(task).__name__ @huey_results.task(include_task=True)
def config_error(msg): print(configuration_message) print('\n\n') print(msg) sys.exit(1) HUEY = getattr(settings, 'HUEY', None) if HUEY is None: try: from huey import RedisHuey except ImportError: config_error('Error: Huey could not import the redis backend. ' 'Install `redis-py`.') else: HUEY = RedisHuey(default_queue_name()) if isinstance(HUEY, dict): huey_config = HUEY.copy() # Operate on a copy. name = huey_config.pop('name', default_queue_name()) conn_kwargs = huey_config.pop('connection', {}) try: del huey_config['consumer'] # Don't need consumer opts here. except KeyError: pass if 'always_eager' not in huey_config: huey_config['always_eager'] = settings.DEBUG huey_config.update(conn_kwargs) HUEY = RedisHuey(name, **huey_config) task = HUEY.task