# -*- coding: utf-8 -*-
import datetime
from urllib import parse
from huey import SqliteHuey
from database.sql_test import session, HSZCOMPANY
from downloader import Downloader
down = Downloader()
huey = SqliteHuey(filename='./manage.db')
# 除非大于本月或者不是才能入这里，所以当为本月的时候说明全部都采集完毕
@huey.task(retries=1)
def start_spider(userId):
result = session.query(HSZCOMPANY).filter(HSZCOMPANY.userId == userId,
HSZCOMPANY.eventuallykjqj != 0).all()
headers = session.query(HSZCOMPANY).filter(HSZCOMPANY.userId == userId,
HSZCOMPANY.headers.like('%{0}%'.format("sap"))).first().headers
headers = dict(parse.parse_qsl(headers))
for zt in result:
url = 'https://sap.kungeek.com/portal/ftsp/portal/khxx.do?handleCustomer'
data = {'ztZtxxId': zt.ztId, 'khxxId': zt.id_num}
print(data)
account_date = down.visit(url=url, response_container=['json', 'utf-8'], method='post', data=data,
headers=headers)
if account_date.get('data', None):
fwqxz = account_date['data']['fwqxz']
startkjqj = account_date['data']['qyQj']
notebooks[key] = path
return notebooks
class HueyConfig(object):
BROKER_URL = os.environ.get('BROKER_URL', '/tmp/db.sqlite')
class AppConfig(object):
DEFAULT_OUTPUT_DESTINATION = os.path.join(
os.path.abspath(os.path.dirname(__name__)),
'jobs'
)
OUTPUT_DESTINATION = os.environ.get(
'OUTPUT_DESTINATION', DEFAULT_OUTPUT_DESTINATION)
if not os.path.exists(OUTPUT_DESTINATION):
os.mkdir(OUTPUT_DESTINATION)
EXAMPLES_DIR = os.path.join(
os.path.dirname(os.path.abspath(__name__)), 'examples')
EXAMPLES_NOTEBOOKS = get_example_notebooks(EXAMPLES_DIR)
huey = SqliteHuey(filename=HueyConfig.BROKER_URL)
logging.config.fileConfig('logging.conf')
logger = logging.getLogger('notebooks')
from huey import SqliteHuey
from os import environ
import sys
data_dir = '{}/data'.format(environ['APP_DIR'])
if 'APP_URL' in environ:
app_url = environ['APP_URL'].strip('/')
else:
app_url = ''
queue_db = '{}/queue.db'.format(data_dir)
huey = SqliteHuey(filename=queue_db)
import logging
from huey import SqliteHuey
from slc.appbootstrap import options
from slc.mailer import get_mailer
logger = logging.getLogger(__name__)
huey = SqliteHuey(filename=options.HUEY_DB)
@huey.task(retries=10, retry_delay=60)
def queue_send_mail(*args, **kwargs):
mailer = get_mailer()
logger.warning(f"Sending via {mailer.username} @ {mailer.hostname}")
mailer.send(*args, **kwargs)
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'cyads_transparency_web.wsgi.application'
HUEY = SqliteHuey('transparency', filename='huey.sqlite3')
# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.environ["TRANSPARENCY_DB_FILE"],
}
}
# Password validation
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
from huey import SqliteHuey
import os
import json
schedule = SqliteHuey('test.db', utc=False)
class Config:
FLASK_DEBUG = os.environ.get('FLASK_DEBUG')
# Database Settings
MONGO_HOST = os.environ.get('MONGO_HOST')
MONGO_PORT = os.environ.get('MONGO_PORT')
# Relic APIs
RELIC_API_BASE_URL = os.environ.get('RELIC_API_BASE_URL')
LEADERBOARDS = os.environ.get('LEADERBOARDS')
SPECIFIC_LEADERBOARD = os.environ.get('SPECIFIC_LEADERBOARD')
PROFILES_STATS = os.environ.get('PROFILES_STATS')
RECENT_MATCH_HISTORY = os.environ.get('RECENT_MATCH_HISTORY')
# Endpoints for screenshots
STATS_1V1_URL = os.environ.get('STATS_1v1_URL')
STATS_TEAMS_URL = os.environ.get('STATS_TEAMS_URL')
# Facebook Settings
FB_GROUP_ID = os.environ.get(
'FB_TEST_GROUP_ID') if FLASK_DEBUG else os.environ.get('FB_GROUP_ID')
FB_TOKEN = os.environ.get('FB_TOKEN')
# Relic Headers
from flask import Flask
from huey import RedisHuey
from huey import SqliteHuey
DEBUG = True
SECRET_KEY = 'shhh, secret'
app = Flask(__name__)
app.config.from_object(__name__)
# huey = RedisHuey()
huey = SqliteHuey(
filename='/Users/xubiao/Downloads/huey-master/examples/flask_ex/demo.db')
import os
from huey import SqliteHuey
huey = SqliteHuey(filename="huey.db", )
@huey.task()
def add(a, b):
print(os.getpid(), f": {a}+{b}={a+b}")
return a + b
'DEFAULT_RENDERER_CLASSES': (
'djangorestframework_camel_case.render.CamelCaseJSONRenderer',
'djangorestframework_camel_case.render.CamelCaseBrowsableAPIRenderer',
),
'DEFAULT_PARSER_CLASSES': (
'djangorestframework_camel_case.parser.CamelCaseFormParser',
'djangorestframework_camel_case.parser.CamelCaseMultiPartParser',
'djangorestframework_camel_case.parser.CamelCaseJSONParser',
),
'JSON_UNDERSCOREIZE': {
'no_underscore_before_number': True,
},
}
# Extra stuff
GEOCODER_API_KEY = GEOCODIO_API_KEY
CORS_ORIGIN_ALLOW_ALL = True
CKEDITOR_CONFIGS = {
'default': {
'uiColor': '#edf4f8',
'removeButtons': 'Flash,Styles,Format,Horizontal Line,Anchor'
}
}
PHONENUMBER_DEFAULT_REGION = 'US'
HUEY = SqliteHuey(filename="/var/www/backend/tmp/huey-db.sqlite3")
HUEY.immediate_use_memory = False
from huey import SqliteHuey huey = SqliteHuey()
from huey import SqliteHuey
from entities import db_name
from utils import encode, decode
from os import rename, remove
import logging
huey = SqliteHuey(filename=db_name)
CHUNK_SIZE = 4096
@huey.task()
def encode_file(filename, key):
with open(filename, "rb") as fi, open(filename + ".enc", "wb") as fo:
while fi.readable():
chunk = fi.read(CHUNK_SIZE)
if len(chunk) == 0:
break
fo.write(encode(chunk, key))
remove(filename)
rename(filename + ".enc", filename)
from huey import SqliteHuey huey_queue = SqliteHuey(filename="huey_queue.db")
from huey import SqliteHuey from .logging import config task_queue = SqliteHuey(config["task_database_path"])
class DatabaseException(Exception):
pass
class ConfigError(Exception):
pass
if 'POSTGIS_BASELAYERS_WORKDIR' not in os.environ:
os.environ['POSTGIS_BASELAYERS_WORKDIR'] = '/opt/postgis-baselayers'
# Create our SqliteHuey instance
huey_file = os.path.join(os.environ['POSTGIS_BASELAYERS_WORKDIR'], 'huey.db')
huey = SqliteHuey(filename=huey_file)
# Function to fetch a database connection
def get_db():
conn = psycopg2.connect(dbname=os.environ.get("POSTGRES_DB", ""),
user=os.environ.get("POSTGRES_USER", ""),
password=os.environ.get("POSTGRES_PASSWORD", ""),
port=os.environ.get("POSTGRES_PORT", 5432),
host=os.environ.get("POSTGRES_HOST", ""))
return conn
# Validate the environment to make sure several environment variables are
# defined, and it sets up the database connectivity on flask's g.conn.
@app.before_request
#!/usr/bin/env python
from huey import SqliteHuey
from django.core.mail import send_mail
from django.template.loader import render_to_string
from twilio.rest import Client
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lottee_new.settings')
huey = SqliteHuey(filename='db.sqlite3')
# from huey import RedisHuey
# from redis import ConnectionPool
# pool = ConnectionPool(host='127.0.0.1', port=6379, max_connections=20)
# huey = RedisHuey('lote', connection_pool=pool)
@huey.task()
def send_code(identifier, code):
import django
django.setup()
# settings.configure()
if '@' in identifier:
html_message = render_to_string(
'email.html',
{
'title': 'Подтверждение почты',
'text': f'Lottee code {code}',
'button_text': 'Подтвердить почту',
'code': code
def get_huey(reset=False):
if reset:
os.remove(huey_file)
return SqliteHuey(filename=huey_file)
from huey import SqliteHuey
from scout_apm.huey import attach_scout
huey = SqliteHuey(filename="db.sqlite3")
# Setup according to https://docs.scoutapm.com/#huey
# Settings done through Django
attach_scout(huey)
@huey.task()
@huey.lock_task("hello")
def hello():
return "Hello World!"
@huey.task()
def fail():
raise ValueError("BØØM!") # non-ASCII
from huey import SqliteHuey from settings import Config huey = SqliteHuey(filename=Config.TASKS_FILE)
import os
from huey import RedisHuey, SqliteHuey
# TODO: refactor getting path
basedir = os.path.abspath(os.path.dirname(__file__))
basedir = '/'.join(basedir.split('/')[:-2])
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'sqlite:///' + os.path.join(basedir, 'WIET_sourcing/appdb.db'))
REDIS_URL = os.getenv('REDIS_URL')
SQLITE_PATH = "sqlite:///" + os.path.join(basedir, 'WIET_sourcing/appdb.db')
huey = RedisHuey(
url=REDIS_URL) if REDIS_URL is not None else SqliteHuey(SQLITE_PATH)
import logging
from huey import SqliteHuey, crontab
huey = SqliteHuey(filename='/tmp/demo.db')
LOGGER = logging.getLogger("huey")
print("handlers in demo.py (before adding our custom handler):",
LOGGER.handlers)
# TODO How to disable altogether the huey default formatter and handler?
# Add our own handler and formatter.
custom_handler = logging.StreamHandler()
custom_logformat = '[%(levelname)s:%(name)s:%(threadName)s] %(message)s'
custom_handler.setFormatter(logging.Formatter(custom_logformat))
LOGGER.addHandler(custom_handler)
print("handlers in demo.py (after adding our custom handler):",
LOGGER.handlers)
@huey.task()
def add(a, b):
return a + b
@huey.periodic_task(crontab(minute='*/1'))
def add_periodic():
res = add(41, 1)
LOGGER.info("Add 41 + 1 = %d", res(blocking=True))
import os
import pendulum
from huey import RedisHuey, SqliteHuey
from .push import send_notification
redis_url = os.environ.get('REDIS_URL')
huey = RedisHuey(url=redis_url) if redis_url else SqliteHuey()
@huey.task()
def remind_subscriber(subscription):
reminder = {
"title": "Another minute passed",
"body": pendulum.now().format('dddd DD MMMM YYYY', locale='es')
}
send_notification(subscription, reminder)
import os
import decouple
from huey import SqliteHuey
from purplship.server.settings import base as settings
# Purplship Server Background jobs interval config
DEFAULT_SCHEDULER_RUN_INTERVAL = 3600 # value is seconds. so 3600 seconds = 1 Hour
DEFAULT_TRACKERS_UPDATE_INTERVAL = 10800 # value is seconds. so 10800 seconds = 3 Hours
WORKER_DB_DIR = decouple.config('WORKER_DB_DIR', default=settings.WORK_DIR)
WORKER_DB_FILE_NAME = os.path.join(WORKER_DB_DIR, 'tasks.sqlite3')
settings.DATABASES['workers'] = {
'NAME': WORKER_DB_FILE_NAME,
'ENGINE': 'django.db.backends.sqlite3',
}
HUEY = SqliteHuey(
name='default',
filename=WORKER_DB_FILE_NAME
)
from huey.signals import SIGNAL_COMPLETE, SIGNAL_ERROR
import os
from huey import SqliteHuey
import subprocess
huey = SqliteHuey('main')
#The following functions are used to startup/shutdown the Huey consumer console.
def startup_consumer():
p = subprocess.Popen("huey_consumer queuemanager.huey",
creationflags=subprocess.CREATE_NEW_CONSOLE)
##shutdown_consumer():
##force_shutdown_consumer():
#The following functions are used to give feedback on tasks in the queue.
@huey.signal(SIGNAL_COMPLETE)
def task_completed(signal, task):
print('%s - %s - %s' % (signal, task.name, task))
print('task was completed.')
@huey.signal(SIGNAL_ERROR)
def task_error(signal, task, exc=None):
print('%s - %s' % (signal, task.name))
print('task error.')
from huey import SqliteHuey
from cloudcopy.server.config import settings
# if ASYNC_TASKS is False (e.g. in testing)
# async tasks will be executed immediately
# or added to an in-memory schedule registry
app = SqliteHuey('tasks',
filename=settings.INTERNAL_DATABASE_FILE,
immediate=not settings.ASYNC_TASKS)
# Selenium Imports
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
# Timers
import random
huey = SqliteHuey(filename='./task.db')
place_id_list = []
API_KEY = 'AIzaSyCCNjBph61HIqSnJ5kgLzapqPYyRtEj9DQ'
user_data_wp = os.path.abspath("./user-data-wp")
global chrome_options
chrome_options = Options()
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument("user-data-dir="+ str(user_data_wp))
global chrome_path
global first_check
from huey import SqliteHuey
huey = SqliteHuey('testing')
@huey.task(retries=10, retry_delay=300)
def schedule_reply(submission, reply_content, reddit):
'''
schedule_reply will add task to reply a given submission via comment
the scheduled task will run outside of main processing window to avoid
excessive reddit API ratelimiting. Tasks run with retries & delays in between
'''
reddit.reply(submission, reply_content)
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # For any questions about this software or licensing, # please email [email protected] or [email protected]. # from huey import SqliteHuey huey = SqliteHuey(filename='s3cluster_queue.db') # Cluster-specific config config_dir = f'/root/perf/configs/' artifacts_dir = f'/var/results' # motr_src_dir should be defined for execute taskq framework # outside motr source directory motr_src_dir = '/root/perf/motr' # fio_test_dir = '/root/perf/fio-test' pack_artifacts = False
import os
import requests
import json
from huey import SqliteHuey
from cody.config import CODY_FOLDER
from cody.shell import execute_commands
huey = SqliteHuey(filename=os.path.join(CODY_FOLDER, '.cody.db'))
@huey.task()
def deploy_app(project_name, path, slack_hook):
(output, exit_code) = execute_commands([os.path.join(path, 'cody.sh')])
if exit_code == 0:
text = "🦊 Project {0}\n✅ Deploy successful".format(project_name)
else:
text = "🦊 Project {0}\n❌ Deploy has failed!\n📄 Exit code {1} - {2}".format(
project_name, exit_code, output)
payload = {'text': text}
print(slack_hook)
r = requests.post(slack_hook, data=json.dumps(payload))
print(r)
import configparser
import os
from huey import SqliteHuey
class Config:
def __init__(self, path='../config.ini'):
config = configparser.ConfigParser()
config.read(path)
self.database = config['Database']
self.plugins = config['Plugins']
self.redis = config['Redis']
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'data', 'tasks.db')
huey = SqliteHuey(filename=filename)
""" Setup for the Huey task queue. """
# pylint: disable=invalid-name,ungrouped-imports,unused-import
from os import environ
# run consumer via:
# huey_consumer.py app.huey
# from the root of the app directory
# ------------------------------------------------------
# Huey task config
# ------------------------------------------------------
# Redis is the preferred Huey backend
REDIS_URL = environ.get('REDIS_URL', None)
if REDIS_URL:
from huey import RedisHuey
huey = RedisHuey(url=REDIS_URL)
# But in dev environments, fall back to SqliteHuey if Redis is not available
else:
try:
import peewee
except ImportError:
MSG = "peewee is required for SqliteHuey."
MSG += "Install peewee directly, don't add to requirements.txt"
raise RuntimeError(MSG)
from huey import SqliteHuey
huey = SqliteHuey(filename='huey.db')
