def __init__(self, options):
BaseQueue.__init__(self, options)
# TODO: We should rethink ``store_errors=False`` -- how do we notice errors?
self._huey = RedisHuey(result_store=False,
store_none=False,
store_errors=False,
**options)
self._jobs = {}
def test_non_global_task_registry(self):
huey = RedisHuey(global_registry=False)
@huey.task()
def test():
return 'test'
self.assertIn('queuecmd_test', huey.registry._registry)
huey2 = RedisHuey(global_registry=False)
self.assertNotIn('queuecmd_test', huey2.registry._registry)
class HueyQueue(BaseQueue):
def __init__(self, options):
BaseQueue.__init__(self, options)
# TODO: We should rethink ``store_errors=False`` -- how do we notice errors?
self._huey = RedisHuey(result_store=False,
store_none=False,
store_errors=False,
**options)
self._jobs = {}
@property
def huey(self):
return self._huey
@property
def jobs(self):
return self._jobs
def register_job(self, name, func):
if name in self._jobs:
raise QueueError(u"Job function {!r} already exists".format(name))
self._jobs[name] = self._huey.task(name=name)(func)
def enqueue(self, name, args, kwargs):
if name not in self._jobs:
raise QueueError(u"Unknown job: {!r}".format(name))
log.info(u"Sending {!r} job to the queue ...".format(name))
# We do not care about results
self._jobs[name](*args, **kwargs)
def init_huey(settings: dict) -> RedisHuey:
global huey
huey = RedisHuey(
url=settings["zam.tasks.redis_url"],
always_eager=asbool(settings.get("zam.tasks.always_eager", "False")),
)
return huey
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
def create_huey():
"""
Huey is our asynchronous task scheduler.
https://huey.readthedocs.io/en/latest/
"""
if settings.PROCESS_ASYNC_TASKS:
return RedisHuey(
connection_pool=cache.RedisCache.connect().connection_pool)
return DummyHuey()
def __init__(self):
self.logger = logging.getLogger('App.App')
self.logger.setLevel(logging.DEBUG)
self.config = config
self.db = DB(config)
self.huey = RedisHuey(host="redis", result_store=False)
self.web3 = Web3(HTTPProvider(config.HTTP_PROVIDER_URL))
self._tokens = None
self.updateTokens()
def __init__(self, name, connection_pool):
self.name = name
self.clean_name = self.clean_queue_name(self.name)
self.huey = RedisHuey(self.name, connection_pool=connection_pool)
self.event_handlers = {
EVENT_FINISHED: self.event_finished,
EVENT_ENQUEUED: self.event_enqueued,
EVENT_STARTED: self.event_started,
EVENT_ERROR_TASK: self.event_error,
}
def _get_asynchronous_result(huey_app: RedisHuey, huey_task_id: str):
try:
huey_task = huey_app.result(huey_task_id)
except TaskException as e:
try:
for import_exception in imported_exceptions:
exec(import_exception)
task_exception = eval(e.metadata["error"])
except NameError:
task_exception = Exception(e.metadata["error"])
raise task_exception
return huey_task
def connect_huey():
global huey
try:
queue_name = get_settings().HUEY_TASK_QUEUES
url = get_settings().HUEY_REDIS_CONNECTION
if url != "" and queue_name != "":
huey = RedisHuey(queue_name, url=url, blocking=False)
print("Huey Connect")
else:
print("Huey No Connection ")
except Exception as e:
print("Huey Connect Fail: " + str(e))
return huey
def build_async_application(config: dict, **kwargs) -> RedisHuey:
"""
This function should be called within a python module called 'asynchronous_server.py'
Build a huey application with given configuration and modules
:param config: Dictionary with following structure: {
'asynchronous': {
'broker': ...,
}
}
:param kwargs: Additional Huey arguments
:return: RedisHuey Application
"""
logger.info(f"Starting Huey server")
return RedisHuey(
os.getenv("CONTAINER_NAME", "LOCAL"),
url=config["asynchronous"]["broker"],
**kwargs,
)
def create_huey(config):
_storage_args = {
'read_timeout': config.get('HUEY_STORAGE_READ_TIMEOUT', 1),
'max_errors': config.get('HUEY_STORAGE_MAX_ERRORS', 1000),
'connection_pool': config.get('HUEY_STORAGE_CONNECTION_POOL', None),
'url': config.get('HUEY_STORAGE_URL', None),
}
_huey = RedisHuey(name=config.get('HUEY_TASK_QUEUE_NAME',
'FLASK-HUEY-EXAMPLE'),
result_store=config.get('HUEY_RESULT_STORE', False),
events=config.get('HUEY_EVENTS', True),
store_none=config.get('HUEY_STORE_NONE', False),
always_eager=config.get('HUEY_ALWAYS_EAGER', False),
store_errors=config.get('HUEY_STORE_ERRORS', True),
blocking=config.get('HUEY_BLOCKING', False),
**_storage_args)
return _huey
import smtplib
import datetime
from huey import RedisHuey, crontab
from . import database, handler, crypto
iterate = handler.Iterate()
webdata = handler.WebData()
quicrypt = crypto.QuiCrypt()
c = handler.Bunch(webdata.loader('config.json'))
try:
huey = RedisHuey('hub-queue', host=c.db_host, port=6379)
except:
huey = RedisHuey('hub-queue', host=c.db_back, port=6379)
class Automation(object):
def __init__(self):
pass
@huey.periodic_task(crontab(minute='*', hour='*'))
def cleanUserDatabase():
'''Huey task to routinely clean the database of all user accounts that are unverified ''' # Set to Five Minutes for debug
conn = database.connect(c.db_name, c.db_host, c.db_port, c.db_back)
# expired = time.time() - 86400 # One day ago
expired = time.time() - 300 # Five minute(s) ago
r = database.custom() # Fetch the entire custom database object - kinda heavy; make sure it's cleaned up
import zhihu
from pymongo import MongoClient
from huey import RedisHuey
from requests.adapters import HTTPAdapter
from zhihu import ANONYMOUS
from common import *
if hasattr(os, '_called_from_test'):
from client_pool import get_client_test as get_client # don't use proxy
else:
from client_pool import get_client_test as get_client # use kunpeng proxy
from utils import config_smtp_handler
huey = RedisHuey()
logger = logging.getLogger('huey.consumer')
logger2 = logging.getLogger('huey.consumer.Worker')
logger2.setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
if os.path.isfile(logging_config_file):
with open(logging_config_file, 'rt') as f:
config = json.load(f)
mail_handler_cfg = config['handlers']['mail_handler']
smtp_handler = logging.handlers.SMTPHandler(
mailhost=(mail_handler_cfg['mailhost'], 25),
fromaddr=mail_handler_cfg['fromaddr'],
toaddrs=mail_handler_cfg['toaddrs'],
subject="Huey Error")
import pickle
from huey import crontab
from huey import exceptions as huey_exceptions
from huey import RedisHuey
from huey.api import Huey
from huey.api import QueueTask
from huey.api import TaskWrapper
from huey.constants import EmptyData
from huey.registry import registry
from huey.storage import RedisStorage
from huey.tests.base import b
from huey.tests.base import BaseTestCase
from huey.utils import local_to_utc
huey = RedisHuey(result_store=False, events=False, blocking=False)
huey_results = RedisHuey(blocking=False, max_errors=10)
huey_store_none = RedisHuey(store_none=True, blocking=False)
# Global state.
state = {}
@huey.task()
def put_data(key, value):
state[key] = value
@huey.task(include_task=True)
def put_data_ctx(key, value, task=None):
state['last_task_class'] = type(task).__name__
@huey_results.task(include_task=True)
import time
from huey import RedisHuey
huey = RedisHuey("predictions", host="localhost", port=16379)
@huey.task()
def predict(features):
time.sleep(5)
s = sum(n for n in features.values())
print("Predicted:", s)
return s
def create_prediction(features):
return predict(features)
from huey import RedisHuey, crontab
from portality.core import app
main_queue = RedisHuey('doaj_main_queue',
host=app.config['HUEY_REDIS_HOST'],
port=app.config['HUEY_REDIS_PORT'],
always_eager=app.config.get("HUEY_EAGER", False))
long_running = RedisHuey('doaj_long_running',
host=app.config['HUEY_REDIS_HOST'],
port=app.config['HUEY_REDIS_PORT'],
always_eager=app.config.get("HUEY_EAGER", False))
def schedule(action):
cfg = app.config.get("HUEY_SCHEDULE", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError(
"No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again."
.format(x=action))
return crontab(**action_cfg)
def configure(action):
cfg = app.config.get("HUEY_TASKS", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError(
"No task configuration for action '{x}'. Define this in HUEY_TASKS first then try again."
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import pystan
import pickle
from huey import RedisHuey
huey = RedisHuey('iris')
def get_iris_dataset():
iris_file = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
return pd.read_csv(iris_file,
names=[
'sepal_length', 'sepal_width', 'petal_length',
'petal_width', 'species'
])
def format_stan_data(iris_data):
# Simple multi-level model, predict petal length using sepal length
# and random effects from each species.
species = LabelEncoder()
iris_data['species_num'] = species.fit_transform(iris_data['species']) + 1
stan_iris_data = {
'n': iris_data.shape[0],
'n_species': max(iris_data['species_num']),
'sepal_length': iris_data['sepal_length'],
'species': iris_data['species_num'],
'petal_length': iris_data['petal_length']
import os
import json
import requests
import random
from huey import RedisHuey
# worker
huey = RedisHuey(url=os.getenv('REDIS_URL'))
# task
@huey.task(retries=5, retry_delay=5)
def get_random_num():
print("This is a task to get a random number")
num = random.randint(1, 3)
print("Random number is {}".format(num))
if num == 1: # if number is 1, the task was successful
return True
else: # if number is not 1, raise an error (so that we see how a failed task looks like and what happens next)
raise Exception("Error in the worker... :(")
# task
@huey.task(retries=10, retry_delay=600)
def send_email_task(receiver_email, subject, text):
sender_email = os.getenv("MY_SENDER_EMAIL") # Your website's official email address
api_key = os.getenv('SENDGRID_API_KEY')
if sender_email and api_key:
url = "https://api.sendgrid.com/v3/mail/send"
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_ROOT = os.path.join(PROJECT_DIR, 'static')
from huey import RedisHuey
from redis import ConnectionPool
import os
huey_app = os.getenv('huey_app', 'production_app')
always_eager = False
if huey_app != 'production_app':
always_eager = False
print('Always eager huey!')
pool = ConnectionPool.from_url(
'redis://*****:*****@ec2-34-252-182-25.eu-west-1.compute.amazonaws.com:27659'
)
HUEY = RedisHuey(huey_app, connection_pool=pool, always_eager=always_eager)
print('huey app:', HUEY.name)
return load_class(path + '.Components')
except ImportError:
config_error('Unable to import %s: "%s"' % (key, path))
try:
HUEY = getattr(settings, 'HUEY', None)
except:
config_error('Error encountered reading settings.HUEY')
if HUEY is None:
try:
from huey import RedisHuey
except ImportError:
config_error('Error: Huey could not import the redis backend. '
'Install `redis-py`.')
HUEY = RedisHuey(default_queue_name())
if not isinstance(HUEY, Huey):
Queue, DataStore, Schedule = dynamic_import(HUEY, 'backend')
name = HUEY.get('name') or default_queue_name()
conn = HUEY.get('connection', {})
always_eager = HUEY.get('always_eager', False)
HUEY = Huey(
Queue(name, **conn),
DataStore(name, **conn),
Schedule(name, **conn),
always_eager=always_eager)
task = HUEY.task
periodic_task = HUEY.periodic_task
import email
from os import getenv
from huey import RedisHuey
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.header import Header
huey = RedisHuey(utc=False,
host=getenv('REDIS_HOST') or '127.0.0.1',
port=int(getenv('REDIS_PORT') or 6379),
db=int(getenv('REDIS_DB') or 0))
@huey.task() # 发送邮件案例
def send_email(content, name, phone, e_mail):
"""发送邮件"""
try:
username = '******' # 发送人邮箱
password = '******' # 发送密钥
replyto = '' # 回复邮件接受人邮箱
receiver = '*****@*****.**' # 接受者邮箱
msg = MIMEMultipart('html')
msg['Subject'] = Header('邮件标题')
from_header = Header('*****@*****.**', 'utf-8') # 来自...
from_header.append('<{}>'.format(username), 'ascii')
msg['From'] = from_header
msg['To'] = receiver
msg['Reply-to'] = replyto
msg['Message-id'] = email.utils.make_msgid()
def get_storage(self, **kwargs):
return BaseStorage()
class BrokenRedisStorage(RedisStorage):
def dequeue(self):
raise ValueError('broken redis dequeue')
broken_redis_storage = BrokenRedisStorage()
class BrokenHuey(Huey):
def get_storage(self):
return broken_redis_storage
dummy_huey = DummyHuey()
test_huey = RedisHuey('testing', blocking=False, read_timeout=0.1)
# Logger used by the consumer.
logger = logging.getLogger('huey.consumer')
logger.addHandler(logging.NullHandler())
# Create a log handler that will track messages generated by the consumer.
class CaptureLogs(logging.Handler):
def __init__(self, *args, **kwargs):
self.messages = []
logging.Handler.__init__(self, *args, **kwargs)
def emit(self, record):
self.messages.append(record.getMessage())
def __enter__(self):
from concurrent.futures import ProcessPoolExecutor, as_completed
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import requests
import botometer
from settings import Settings
from elasticsearch import Elasticsearch
from huey import RedisHuey
import re
import sys
STEP = 100
TWITTER_DATETIME_PATTERN = "ddd MMM DD HH:mm:SS Z YYYY"
FIXED_TWITTER_DATE_TIME = re.compile('\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z')
MAX_DELTA_BOTOMETER = 3600 * 24 * 2 # 2 Days
huey = RedisHuey('esK3K2')
def esK3K2_ascii_art():
print(" __ _______ __ _____ ")
print(" ___ _____/ //_/__ // //_/__ \ ")
print(" / _ \/ ___/ ,< /_ </ ,< __/ / ")
print("/ __(__ ) /| |___/ / /| |/ __/ ")
print("\___/____/_/ |_/____/_/ |_/____/ ")
print(" ")
def dotter(d: dict, key, dots):
"""Creates a list of key of a dict is very useful to use with Scalpl
Arguments:
d {dict} -- Input dictionary
key {[type]} -- Recursive keys
import os
from huey import RedisHuey
huey_queue = RedisHuey("licenta", host="localhost")
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(ROOT_DIR, 'db.json')
from huey import RedisHuey
huey = RedisHuey('distributor', host='localhost')
from huey import RedisHuey huey = RedisHuey()
import pickle
import requests
from bs4 import BeautifulSoup
from huey import RedisHuey, crontab
from pushbullet import Pushbullet
from redis import StrictRedis, BlockingConnectionPool
url = os.getenv('URL', 'https://shop.lego.com/en-CH/Millennium-Falcon-75192')
redis_host = os.getenv('REDIS_HOST', 'localhost')
redis_port = os.getenv('REDIS_PORT', 6379)
redis_db = os.getenv('REDIS_DB', 0)
pushbullet_api_key = os.getenv('PUSHBULLET_API_KEY', '')
pool = BlockingConnectionPool(host=redis_host, port=redis_port, db=redis_db)
redis = StrictRedis(connection_pool=pool)
huey = RedisHuey('millennium-falcon-checker', connection_pool=pool)
pb = Pushbullet(pushbullet_api_key)
@huey.periodic_task(crontab(minute='*/30'))
def check_status_task():
check_status()
def check_status():
millennium_falcon_response = requests.get(url)
try:
millennium_falcon_response.raise_for_status()
soup = BeautifulSoup(millennium_falcon_response.content, 'html.parser')
if is_available(soup):
print(' Millennium Falcon is available')
from huey import RedisHuey, crontab
from scrape import get_data_send_mail
queue = RedisHuey("scrape-trending")
@queue.periodic_task(crontab(minute="0", hour="*"))
def send_mail():
get_data_send_mail()
import datetime
from huey import RedisHuey, crontab
from db.models import init_db, dbsession
from db.models import Source
from common.scrapers import flipp
init_db()
huey = RedisHuey('ffd_cron', host='redis', immediate_use_memory=True)
@huey.periodic_task(crontab(minute='*'))
def pull_sources():
yesterday = datetime.date.today() - datetime.timedelta(days=7)
source = Source.find_by_method('flipp').filter(
Source.last_updated < yesterday).first()
if source:
meta = source.meta.copy()
meta['source_id'] = source.id
meta['postal_code'] = source.store.postal_code
meta['chain'] = source.store.chain.name
flipp.scrape(meta)
import random
import requests
import smtplib
import time
from ipaddress import ip_address
import tldextract
from github import Github
from huey import RedisHuey, crontab
from pymongo import errors, DESCENDING, ASCENDING
from config.database import result_col, query_col, blacklist_col, notice_col, github_col, setting_col, REDIS_HOST, \
REDIS_PORT
from utils.date import timestamp
from utils.log import logger
from utils.notice import mail_notice
huey = RedisHuey('hawkeye', host=REDIS_HOST, port=int(REDIS_PORT))
base_path = os.path.split(os.path.realpath(__file__))[0]
extract = tldextract.TLDExtract(cache_file='{}/.tld_set'.format(base_path))
if setting_col.count({
'key': 'task',
'minute': {
'$exists': True
},
'page': {
'$exists': True
}
}):
minute = int(setting_col.find_one({'key': 'task'}).get('minute'))
setting_col.update_one(
{'key': 'task'},
from huey import RedisHuey, crontab
# from opennem.api.exporter import wem_run_all
from opennem.settings import settings
huey = RedisHuey("opennem.exporter", host=settings.cache_url.host)
# @huey.periodic_task(crontab(minute="*/10"))
# def wem_export_task():
# wem_run_all()
import datetime
import pickle
from huey import crontab
from huey import exceptions as huey_exceptions
from huey import RedisHuey
from huey.api import Huey
from huey.api import QueueTask
from huey.registry import registry
from huey.storage import RedisStorage
from huey.tests.base import b
from huey.tests.base import BaseTestCase
from huey.utils import EmptyData
from huey.utils import local_to_utc
huey = RedisHuey(result_store=False, events=False, blocking=False)
huey_results = RedisHuey(blocking=False, max_errors=10)
huey_store_none = RedisHuey(store_none=True, blocking=False)
# Global state.
state = {}
@huey.task()
def put_data(key, value):
state[key] = value
@huey.task(include_task=True)
def put_data_ctx(key, value, task=None):
state['last_task_class'] = type(task).__name__
@huey_results.task(include_task=True)
def config_error(msg):
print(configuration_message)
print('\n\n')
print(msg)
sys.exit(1)
HUEY = getattr(settings, 'HUEY', None)
if HUEY is None:
try:
from huey import RedisHuey
except ImportError:
config_error('Error: Huey could not import the redis backend. '
'Install `redis-py`.')
else:
HUEY = RedisHuey(default_queue_name())
if isinstance(HUEY, dict):
huey_config = HUEY.copy() # Operate on a copy.
name = huey_config.pop('name', default_queue_name())
conn_kwargs = huey_config.pop('connection', {})
try:
del huey_config['consumer'] # Don't need consumer opts here.
except KeyError:
pass
if 'always_eager' not in huey_config:
huey_config['always_eager'] = settings.DEBUG
huey_config.update(conn_kwargs)
HUEY = RedisHuey(name, **huey_config)
task = HUEY.task
