def test_crontab_spec_hour_formats(self):
c = crontab(hour=6)
self.assertEqual(c.hour, set([6]))
c = crontab(hour='5')
self.assertEqual(c.hour, set([5]))
c = crontab(hour=(4, 8, 12))
self.assertEqual(c.hour, set([4, 8, 12]))
def setup_periodic_tasks(sender, **kwargs):
sender.add_periodic_task(
3600,
billing_check.s(),
name='billing-check',
)
sender.add_periodic_task(
3600 * 24,
billing_alert.s(),
name='billing-alert',
)
sender.add_periodic_task(
3600 * 24,
billing_notify.s(),
name='billing-notify',
)
sender.add_periodic_task(
crontab(hour=1, minute=0, day_of_week='monday,thursday'),
perform_removal.s(),
name='perform-removal',
)
sender.add_periodic_task(
crontab(hour=2, minute=0, day_of_week='monday,thursday'),
schedule_removal.s(),
name='schedule-removal',
)
sender.add_periodic_task(
crontab(hour=2, minute=30, day_of_week='monday,thursday'),
notify_expired.s(),
name='notify-expired',
)
def conf_frontend_celery(app):
broker = get_frontend_broker_uri()
queue = frontend_config.broker_frontend.queue
_conf_celery(app, broker, queue=queue)
# add celerybeat conf only for frontend app
cron_age_cfg = frontend_config['cron_clean_file_age']
cron_size_cfg = frontend_config['cron_clean_file_size']
app.conf.update(
CELERYBEAT_SCHEDULE={
# File System clean according to file max age
'clean_fs_age': {
'task': 'frontend_app.clean_fs_age',
'schedule': crontab(
hour=cron_age_cfg['clean_fs_age_cron_hour'],
minute=cron_age_cfg['clean_fs_age_cron_minute'],
day_of_week=cron_age_cfg['clean_fs_age_cron_day_of_week']
),
'args': (),
},
# File System clean according to sum max size
'clean_fs_size': {
'task': 'frontend_app.clean_fs_size',
'schedule': crontab(
hour=cron_size_cfg['clean_fs_size_cron_hour'],
minute=cron_size_cfg['clean_fs_size_cron_minute'],
day_of_week=cron_size_cfg['clean_fs_size_cron_day_of_week']
),
'args': (),
},
},
CELERY_TIMEZONE='UTC'
)
def __init__(self, record):
self.name = record.name
self.task = record.task
if record.interval:
i = record.interval
self.schedule = schedules.schedule(timedelta(**{i.period: i.every}))
else:
i = record.crontab
schedules.crontab(minute=i.minute,
hour=i.hour,
day_of_week=i.day_of_week)
try:
self.args = deserialize(record.args or u"[]")
self.kwargs = deserialize(record.kwargs or u"{}")
except ValueError:
record.update_record(no_changes = True, enabled = False)
db(db.celery_periodictasks).update(last_update=datetime.now())
db.commit()
raise
self.options = {"queue": record.queue,
"exchange": record.exchange,
"routing_key": record.routing_key,
"expires": record.expires}
self.total_run_count = record.total_run_count or 0
self.record = record
if not record.last_run_at:
record.update_record(last_run_at = datetime.now())
db(db.celery_periodictasks).update(last_update=datetime.now())
db.commit()
self.last_run_at = record.last_run_at
def test_crontab_spec_dom_formats(self):
c = crontab(day_of_month=5)
self.assertEqual(c.day_of_month, set([5]))
c = crontab(day_of_month='5')
self.assertEqual(c.day_of_month, set([5]))
c = crontab(day_of_month='2,4,6')
self.assertEqual(c.day_of_month, set([2, 4, 6]))
c = crontab(day_of_month='*/5')
self.assertEqual(c.day_of_month, set([1, 6, 11, 16, 21, 26, 31]))
def test_crontab_spec_minute_formats(self):
c = crontab(minute=30)
self.assertEqual(c.minute, set([30]))
c = crontab(minute='30')
self.assertEqual(c.minute, set([30]))
c = crontab(minute=(30, 40, 50))
self.assertEqual(c.minute, set([30, 40, 50]))
c = crontab(minute=set([30, 40, 50]))
self.assertEqual(c.minute, set([30, 40, 50]))
def setup_periodic_tasks(sender, **kwargs):
sender.add_periodic_task(
3600,
commit_pending.s(),
name='commit-pending',
)
sender.add_periodic_task(
crontab(hour=3, minute=30),
update_remotes.s(),
name='update-remotes',
)
sender.add_periodic_task(
3600 * 24,
repository_alerts.s(),
name='repository-alerts',
)
sender.add_periodic_task(
3600 * 24,
component_alerts.s(),
name='component-alerts',
)
sender.add_periodic_task(
3600 * 24,
cleanup_suggestions.s(),
name='suggestions-cleanup',
)
sender.add_periodic_task(
3600 * 24,
cleanup_stale_repos.s(),
name='cleanup-stale-repos',
)
sender.add_periodic_task(
3600 * 24,
cleanup_old_suggestions.s(),
name='cleanup-old-suggestions',
)
sender.add_periodic_task(
3600 * 24,
cleanup_old_comments.s(),
name='cleanup-old-comments',
)
# Following fulltext maintenance tasks should not be
# executed at same time
sender.add_periodic_task(
crontab(hour=2, minute=30, day_of_week='saturday'),
cleanup_fulltext.s(),
name='fulltext-cleanup',
)
sender.add_periodic_task(
crontab(hour=2, minute=30, day_of_week='sunday'),
optimize_fulltext.s(),
name='fulltext-optimize',
)
def test_crontab_spec_moy_formats(self):
c = crontab(month_of_year=1)
self.assertEqual(c.month_of_year, set([1]))
c = crontab(month_of_year='1')
self.assertEqual(c.month_of_year, set([1]))
c = crontab(month_of_year='2,4,6')
self.assertEqual(c.month_of_year, set([2, 4, 6]))
c = crontab(month_of_year='*/2')
self.assertEqual(c.month_of_year, set([1, 3, 5, 7, 9, 11]))
c = crontab(month_of_year='2-12/2')
self.assertEqual(c.month_of_year, set([2, 4, 6, 8, 10, 12]))
def init_app(app):
celery.main = app.import_name
celery.conf.update(app.config)
celery.conf['CELERYBEAT_SCHEDULE'] = {
'bump-metrics-every-nights': {
'task': 'bump-metrics',
'schedule': crontab(hour=0, minute=0),
},
'purge-datasets-every-nights': {
'task': 'purge-datasets',
'schedule': crontab(hour=0, minute=0),
},
'purge-reuses-every-nights': {
'task': 'purge-reuses',
'schedule': crontab(hour=0, minute=0),
},
'purge-organizations-every-nights': {
'task': 'purge-organizations',
'schedule': crontab(hour=0, minute=0),
},
}
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
# Load core tasks
import udata.core.metrics.tasks
import udata.core.storages.tasks
# import udata.core.search.tasks
import udata.core.activity.tasks
import udata.core.dataset.tasks
import udata.core.reuse.tasks
import udata.core.organization.tasks
# Load plugins tasks
for plugin in app.config['PLUGINS']:
name = 'udata.ext.{0}.tasks'.format(plugin)
try:
__import__(name)
except ImportError:
pass
except Exception as e:
log.error('Error importing %s: %s', name, e)
return celery
def create_app(config={}):
app = Flask('aleph')
app.config.from_object(settings)
app.config.update(config)
if 'postgres' not in settings.DATABASE_URI:
raise RuntimeError("aleph database must be PostgreSQL!")
app.config.update({
'SQLALCHEMY_DATABASE_URI': settings.DATABASE_URI,
'BABEL_DOMAIN': 'aleph'
})
queue = Queue(settings.QUEUE_NAME,
routing_key=settings.QUEUE_ROUTING_KEY,
queue_arguments={'x-max-priority': 9})
celery.conf.update(
imports=('aleph.queues'),
broker_url=settings.BROKER_URI,
task_always_eager=settings.EAGER,
task_eager_propagates=True,
task_ignore_result=True,
task_acks_late=True,
task_queues=(queue,),
task_default_queue=settings.QUEUE_NAME,
task_default_routing_key=settings.QUEUE_ROUTING_KEY,
worker_max_tasks_per_child=1000,
result_persistent=False,
beat_schedule={
'hourly': {
'task': 'aleph.logic.scheduled.hourly',
'schedule': crontab(hour='*', minute=0)
},
'daily': {
'task': 'aleph.logic.scheduled.daily',
'schedule': crontab(hour=5, minute=0)
}
},
)
migrate.init_app(app, db, directory=settings.ALEMBIC_DIR)
configure_oauth(app)
mail.init_app(app)
db.init_app(app)
babel.init_app(app)
CORS(app, origins=settings.CORS_ORIGINS)
# This executes all registered init-time plugins so that other
# applications can register their behaviour.
for plugin in get_extensions('aleph.init'):
plugin(app=app)
return app
def test_run_all_due_tasks_after_restart(self):
scheduler_class, shelve = create_persistent_scheduler_w_call_logging()
shelve['tz'] = 'UTC'
shelve['utc_enabled'] = True
shelve['__version__'] = __version__
cur_seconds = 20
def now_func():
return datetime(2018, 1, 1, 1, 11, cur_seconds)
app_schedule = {
'first_missed': {'schedule': crontab(
minute='*/10', nowfun=now_func), 'task': 'first_missed'},
'second_missed': {'schedule': crontab(
minute='*/1', nowfun=now_func), 'task': 'second_missed'},
'non_missed': {'schedule': crontab(
minute='*/13', nowfun=now_func), 'task': 'non_missed'}
}
shelve['entries'] = {
'first_missed': beat.ScheduleEntry(
'first_missed', 'first_missed',
last_run_at=now_func() - timedelta(minutes=2),
total_run_count=10,
schedule=app_schedule['first_missed']['schedule']),
'second_missed': beat.ScheduleEntry(
'second_missed', 'second_missed',
last_run_at=now_func() - timedelta(minutes=2),
total_run_count=10,
schedule=app_schedule['second_missed']['schedule']),
'non_missed': beat.ScheduleEntry(
'non_missed', 'non_missed',
last_run_at=now_func() - timedelta(minutes=2),
total_run_count=10,
schedule=app_schedule['non_missed']['schedule']),
}
self.app.conf.beat_schedule = app_schedule
scheduler = scheduler_class(self.app)
max_iter_number = 5
for i in range(max_iter_number):
delay = scheduler.tick()
if delay > 0:
break
assert {'first_missed', 'second_missed'} == {
item['task'] for item in scheduler.sent}
# ensure next call on the beginning of next min
assert abs(60 - cur_seconds - delay) < 1
def test_eq(self):
self.assertEqual(crontab(day_of_week="1, 2"),
crontab(day_of_week="1-2"))
self.assertEqual(crontab(minute="1", hour="2", day_of_week="5"),
crontab(minute="1", hour="2", day_of_week="5"))
self.assertNotEqual(crontab(minute="1"), crontab(minute="2"))
self.assertFalse(object() == crontab(minute="1"))
self.assertFalse(crontab(minute="1") == object())
def test_crontab(self):
d = self.crontab()
result = self.loads(json.dumps(d))
d.pop('__type__')
self.assertEqual(result, crontab())
def tick():
metrics = db.session.query(models.Metric).all()
for metric in metrics:
cron = schedules.crontab(
minute=metric.cron_minute or '*',
hour=metric.cron_hour or '*',
day_of_week=metric.cron_dow or '*',
day_of_month=metric.cron_dom or '*',
month_of_year=metric.cron_moy or '*')
last_run = metric.last_run
if not last_run:
last_run = datetime.datetime.fromtimestamp(0)
is_due, next_time_to_run = cron.is_due(last_run)
if is_due:
fn = None
if metric.source.source_type.name == 'google analytics':
fn = ga.ga
elif metric.source.source_type.name == 'database':
fn = database.database
else:
raise Exception('bad source type')
fn.delay(metric.id, datetime.datetime.utcnow(), None)
metric.last_run = datetime.datetime.utcnow()
db.session.commit()
def test_monthday_not_weekdaymonthyear(self):
next = self.next_ocurrance(crontab(minute=[5, 42],
day_of_week="mon",
day_of_month=29,
month_of_year="2-4"),
datetime(2010, 1, 29, 0, 5, 15))
self.assertEqual(next, datetime(2010, 3, 29, 0, 5))
def test_default_crontab_spec(self):
c = crontab()
self.assertEqual(c.minute, set(range(60)))
self.assertEqual(c.hour, set(range(24)))
self.assertEqual(c.day_of_week, set(range(7)))
self.assertEqual(c.day_of_month, set(range(1, 32)))
self.assertEqual(c.month_of_year, set(range(1, 13)))
def test_not_weekmonthdayyear(self):
next = self.next_ocurrance(crontab(minute=[5, 42],
day_of_week='fri,sat',
day_of_month=29,
month_of_year='2-10'),
datetime(2010, 1, 28, 14, 30, 15))
self.assertEqual(next, datetime(2010, 5, 29, 0, 5))
def setup_periodic_tasks(sender, **kwargs):
# Run ssdeep match analytic
# Executes every morning at 2:00 a.m.
sender.add_periodic_task(
crontab(hour=2, minute=0),
ssdeep_compare_celery.s(),
)
# Delete old metricbeat indices
# Executes every morning at 3:00 a.m.
metricbeat_enabled = es_storage_config.get('metricbeat_enabled', True)
if metricbeat_enabled:
sender.add_periodic_task(
crontab(hour=3, minute=0),
metricbeat_rollover.s(days=es_storage_config.get('metricbeat_rollover_days')),
)
def test_weekday_monthday(self):
next = self.next_ocurrance(crontab(minute=30,
hour=14,
day_of_week='mon',
day_of_month=18),
datetime(2010, 1, 18, 14, 30, 15))
self.assertEqual(next, datetime(2010, 10, 18, 14, 30))
def schedule(self):
return schedules.crontab(
minute=self.minute,
hour=self.hour,
day_of_week=self.day_of_week,
day_of_month=self.day_of_month,
month_of_year=self.month_of_year)
def test_scheduled_task(self):
tq = TaskQueue('test')
tq.bind_redis(self.conn1)
kw = {
'request': {'method': 'GET',
'url': 'http://httpbin.org'},
'schedule': schedules.crontab('*/10', '1,2-10')
}
self.assertRaises(TaskCNameRequired, tq.add_task, **kw)
kw['cname'] = 'crontest'
task00 = tq.add_task(**kw)
metakey = tq._TaskQueue__metakey(task00['id'])
self.assertEqual(not_bytes(self.conn1.hget(metakey, 'schedule')),
'"*/10 1,2-10 * * *"')
kw['schedule'] = schedules.schedule(30)
kw['cname'] = 'schedtest'
task01 = tq.add_task(**kw)
metakey = tq._TaskQueue__metakey(task01['id'])
self.assertEqual(not_bytes(self.conn1.hget(metakey, 'schedule')),
'"every 30.0 seconds"')
task10 = tq.get_task_by_cname('crontest')
self.assertEqual(task00, task10)
task11 = tq.get_task_by_cname('schedtest')
self.assertEqual(task01, task11)
task = tq._get_task_by_cname('schedtest')
task.dispatch()
task21 = tq.get_task_by_cname('schedtest')
self.assertNotEqual(task11, task21)
now = utcnow()
self.assertTrue(now - timedelta(5) < task21['last_run_at'] < now)
task11.pop('last_run_at')
task11.pop('uuid')
task21.pop('last_run_at')
task21.pop('uuid')
self.assertEqual(task11, task21)
def __init__(self, slug, label, func, minute='*', hour='*', day_of_week='*', day_of_month='*', month_of_year='*'):
self.slug = slug
self.label = label
self.func = func
self.schedule = crontab(
minute=minute, hour=hour, day_of_week=day_of_week,
day_of_month=day_of_month, month_of_year=month_of_year,
)
app.conf.CELERYBEAT_SCHEDULE.update(
{
self.get_task_name(): {
'task': 'mayan_statistics.tasks.task_execute_statistic',
'schedule': self.schedule,
'args': (self.slug,)
},
}
)
app.conf.CELERY_ROUTES.update(
{
self.get_task_name(): {
'queue': 'statistics'
},
}
)
self.__class__._registry[slug] = self
def test_monthyear_not_weekmonthday(self):
next = self.next_ocurrance(crontab(minute=[5, 42],
day_of_week='mon',
day_of_month=29,
month_of_year='2-4'),
datetime(2010, 2, 28, 0, 5, 15))
self.assertEqual(next, datetime(2010, 3, 29, 0, 5))
def setup_scheduler(self, app):
self.app = app
self.app.conf.beat_schedule = {}
self.m1 = self.create_model_interval(
schedule(timedelta(seconds=10)))
self.m1.save()
self.m1.refresh_from_db()
self.m2 = self.create_model_interval(
schedule(timedelta(minutes=20)))
self.m2.save()
self.m2.refresh_from_db()
self.m3 = self.create_model_crontab(
crontab(minute='2,4,5'))
self.m3.save()
self.m3.refresh_from_db()
self.m4 = self.create_model_solar(
solar('solar_noon', 48.06, 12.86))
self.m4.save()
self.m4.refresh_from_db()
# disabled, should not be in schedule
m5 = self.create_model_interval(
schedule(timedelta(seconds=1)))
m5.enabled = False
m5.save()
self.s = self.Scheduler(app=self.app)
def test_weekday_not_monthdayyear(self):
next = self.next_ocurrance(crontab(minute=[5, 42],
day_of_week='mon',
day_of_month=18,
month_of_year='2-10'),
datetime(2010, 1, 11, 0, 5, 15))
self.assertEqual(next, datetime(2010, 10, 18, 0, 5))
def test_simple_crontab_spec(self):
c = crontab(minute=30)
self.assertEqual(c.minute, set([30]))
self.assertEqual(c.hour, set(range(24)))
self.assertEqual(c.day_of_week, set(range(7)))
self.assertEqual(c.day_of_month, set(range(1, 32)))
self.assertEqual(c.month_of_year, set(range(1, 13)))
def includeme(config):
# Register whatever file storage backend has been configured for storing
# our package files.
storage_class = config.maybe_dotted(
config.registry.settings["files.backend"],
)
config.register_service_factory(storage_class.create_service, IFileStorage)
# Register our service which will handle get the download statistics for
# a project.
config.register_service(
RedisDownloadStatService(
config.registry.settings["download_stats.url"],
),
IDownloadStatService,
)
# Register our origin cache keys
config.register_origin_cache_keys(
Project,
cache_keys=["project/{obj.normalized_name}"],
purge_keys=["project/{obj.normalized_name}", "all-projects"],
)
config.register_origin_cache_keys(
Release,
cache_keys=["project/{obj.project.normalized_name}"],
purge_keys=["project/{obj.project.normalized_name}", "all-projects"],
)
# Add a periodic task to compute trending once a day, assuming we have
# been configured to be able to access BigQuery.
if config.get_settings().get("warehouse.trending_table"):
config.add_periodic_task(crontab(minute=0, hour=3), compute_trending)
def includeme(config):
# Register whatever file storage backend has been configured for storing
# our package files.
files_storage_class = config.maybe_dotted(config.registry.settings["files.backend"])
config.register_service_factory(files_storage_class.create_service, IFileStorage)
docs_storage_class = config.maybe_dotted(config.registry.settings["docs.backend"])
config.register_service_factory(docs_storage_class.create_service, IDocsStorage)
# Register our origin cache keys
config.register_origin_cache_keys(
File,
cache_keys=["project/{obj.release.project.normalized_name}"],
purge_keys=[key_factory("project/{obj.release.project.normalized_name}")],
)
config.register_origin_cache_keys(
Project,
cache_keys=["project/{obj.normalized_name}"],
purge_keys=[
key_factory("project/{obj.normalized_name}"),
key_factory("user/{itr.username}", iterate_on="users"),
key_factory("all-projects"),
],
)
config.register_origin_cache_keys(
Release,
cache_keys=["project/{obj.project.normalized_name}"],
purge_keys=[
key_factory("project/{obj.project.normalized_name}"),
key_factory("user/{itr.username}", iterate_on="project.users"),
key_factory("all-projects"),
],
)
config.register_origin_cache_keys(
Role,
purge_keys=[
key_factory("user/{obj.user.username}"),
key_factory("project/{obj.project.normalized_name}"),
],
)
config.register_origin_cache_keys(User, cache_keys=["user/{obj.username}"])
config.register_origin_cache_keys(
User.name,
purge_keys=[
key_factory("user/{obj.username}"),
key_factory("project/{itr.normalized_name}", iterate_on="projects"),
],
)
config.register_origin_cache_keys(
Email.primary,
purge_keys=[
key_factory("user/{obj.user.username}"),
key_factory("project/{itr.normalized_name}", iterate_on="user.projects"),
],
)
# Add a periodic task to compute trending once a day, assuming we have
# been configured to be able to access BigQuery.
if config.get_settings().get("warehouse.trending_table"):
config.add_periodic_task(crontab(minute=0, hour=3), compute_trending)
def test_monthdayyear_not_week(self):
next = self.next_ocurrance(crontab(minute=[5, 42],
day_of_week='wed,thu',
day_of_month=29,
month_of_year='1,4,7'),
datetime(2010, 1, 29, 14, 30, 15))
self.assertEqual(next, datetime(2010, 4, 29, 0, 5))
def test_crontab_spec_invalid_dow(self):
with self.assertRaises(ValueError):
crontab(day_of_week='fooday-barday')
with self.assertRaises(ValueError):
crontab(day_of_week='1,4,foo')
with self.assertRaises(ValueError):
crontab(day_of_week='7')
with self.assertRaises(ValueError):
crontab(day_of_week='12')
CELERY_IMPORTS = config['celery']['tasks']
BROKER_URL = config['heatmaps']['transport']+\
"://"+brokerHostStr(config['heatmaps']['broker'])+\
"/"+str(config['heatmaps']['broker']['database'])
CELERY_RESULT_BACKEND = config['heatmaps']['transport']
CELERY_MONGODB_BACKEND_SETTINGS = config['heatmaps']['broker']
CELERY_ACCEPT_CONTENT = ['pickle', 'json']
## periodic tasks
CELERYBEAT_SCHEDULE = {}
if config["heatmaps"]["enabled"]:
CELERYBEAT_SCHEDULE['heat-queries'] = {
'task': 'metrilyx.celerytasks.run_heat_queries',
'schedule': crontab(minute='*/1'),
#'args': (1,2),
#'options': { 'task_id': '' }
}
if config["cache"]["enabled"]:
CELERYBEAT_SCHEDULE['metric-cacher'] = {
'task': 'metrilyx.celerytasks.cache_metrics',
'schedule': crontab(minute=str("*/%d" %(config['cache']['interval'])))
}
## TODO: re-work logic
#CELERYBEAT_SCHEDULE['metric-cache-expirer'] = {
# 'task': 'metrilyx.celerytasks.expire_metrics_cache',
# 'schedule': crontab(minute=str("*/%d" %(config['cache']['retention_period'])))
#}
if not TESTING:
CELERY_BROKER_URL = os.getenv('REDIS_URL', f'redis://{redis_host}/1')
else:
OPENWISP_RADIUS_GROUPCHECK_ADMIN = True
OPENWISP_RADIUS_GROUPREPLY_ADMIN = True
OPENWISP_RADIUS_USERGROUP_ADMIN = True
CELERY_TASK_ALWAYS_EAGER = True
CELERY_TASK_EAGER_PROPAGATES = True
CELERY_BROKER_URL = 'memory://'
TEST_RUNNER = 'openwisp_utils.tests.TimeLoggingTestRunner'
CELERY_BEAT_SCHEDULE = {
'deactivate_expired_users': {
'task': 'openwisp_radius.tasks.cleanup_stale_radacct',
'schedule': crontab(hour=0, minute=0),
'args': None,
'relative': True,
},
'delete_old_users': {
'task': 'openwisp_radius.tasks.delete_old_users',
'schedule': crontab(hour=0, minute=10),
'args': [365],
'relative': True,
},
'cleanup_stale_radacct': {
'task': 'openwisp_radius.tasks.cleanup_stale_radacct',
'schedule': crontab(hour=0, minute=20),
'args': [365],
'relative': True,
},
},
},
'schedule-deletions': {
'task': 'sentry.tasks.deletion.run_scheduled_deletions',
'schedule': timedelta(minutes=15),
'options': {
'expires': 60 * 25,
},
},
'schedule-weekly-organization-reports': {
'task':
'sentry.tasks.reports.prepare_reports',
'schedule':
crontab(
minute=0,
hour=12, # 05:00 PDT, 09:00 EDT, 12:00 UTC
day_of_week='monday',
),
'options': {
'expires': 60 * 60 * 3,
},
},
'schedule-vsts-integration-subscription-check': {
'task': 'sentry.tasks.integrations.kickoff_vsts_subscription_check',
'schedule': timedelta(hours=6),
'options': {
'expires': 60 * 25,
}
}
}
from celery.schedules import crontab
from celery.task import periodic_task
from celery.utils.log import get_task_logger
from dimagi.utils.dates import DateSpan
from corehq.apps.data_analytics.gir_generator import GIRTableGenerator
from corehq.apps.data_analytics.malt_generator import MALTTableGenerator
from corehq.util.log import send_HTML_email
from corehq.util.soft_assert import soft_assert
logger = get_task_logger(__name__)
@periodic_task(queue='background_queue',
run_every=crontab(hour=1, minute=0, day_of_month='2'),
acks_late=True,
ignore_result=True)
def build_last_month_MALT():
def _last_month_datespan():
today = datetime.date.today()
first_of_this_month = datetime.date(day=1,
month=today.month,
year=today.year)
last_month = first_of_this_month - datetime.timedelta(days=1)
return DateSpan.from_month(last_month.month, last_month.year)
last_month = _last_month_datespan()
generator = MALTTableGenerator([last_month])
generator.build_table()
# Create your tasks here
from __future__ import absolute_import, unicode_literals
from celery import shared_task
from celery.schedules import crontab
from celery.task import periodic_task
@periodic_task(run_every=crontab(hour=14, minute=25))
def every_day_midnight():
print("This is run every Monday morning at 7:30")
for i in range(len(picture_soup)):
data_photo_url = picture_soup[i]["href"]
list_of_pictures.append(data_photo_url)
except Exception as e:
sys.stdout.write("Does not contain any images")
sys.stdout.write(str(e))
if len(list_of_pictures) == 0:
list_of_pictures.append("no images")
# sys.stdout.write("I ended to get data from the site, now i will put it to database")
# Put data into db
self.func_list.get(self.help_model_name, None)(id, characteristics, advert_title, url, phones_data, city, price,
text_data, list_of_pictures)
@periodic_task(run_every=crontab(hour=3, minute=00, day_of_week='tue,fri'))
def main():
print("Print from Scrape main")
ms = MainScrape()
redis_shops = redis.Redis(host='redis', port=6379, db=0)
rediska = redis.Redis(host='redis', port=6379, db=1)
redis_list = [redis_shops, rediska]
for rediska in redis_list:
for link_id in rediska.keys():
link = os.path.join('/a/show', link_id.decode("utf-8"))
try:
help_model_name = rediska.get(link_id.decode("utf-8"))
print("Before first ERRRR...")
if_exists = appconf.CORRESPOND_MODELS[help_model_name.decode("utf-8")].objects.filter \
(link_id=link_id.decode("utf-8")).exists()
except Exception as e:
"ibutsu_server.tasks.db",
"ibutsu_server.tasks.importers",
"ibutsu_server.tasks.reports",
"ibutsu_server.tasks.results",
"ibutsu_server.tasks.runs",
],
)
app.config_from_object(settings)
app.Task = IbutsuTask
# Shortcut for the decorator
task = app.task
# Add in any periodic tasks
app.conf.beat_schedule = {
"prune-old-artifact-files": {
"task": "ibutsu_server.tasks.db.prune_old_files",
"schedule": crontab(minute=0, hour=4,
day_of_week=6), # 4 am on Saturday, after DB dump
"args": (3, ), # delete any artifact file older than 3 months
}
}
@signals.task_failure.connect
def retry_task_on_exception(*args, **kwargs):
"""Retry a task automatically when it fails"""
task = kwargs.get("sender")
einfo = kwargs.get("einfo")
logging.warning("Uncaught exception: %r for task %s", einfo, task)
# Incremental backoff, starts at a minute and maxes out at 1 hour.
backoff = min(2**task.request.retries, 3600)
task.retry(countdown=backoff)
logger = get_task_logger(__name__)
bstamp = Bitstamp()
satang = Satang()
line = Line()
tg = Telegram()
fixer = Fixer()
mckee_app.conf.beat_schedule = {
"""
Check the rate spread every 10 minutes
Update the fx rate every hour
"""
'check-rate-spread': {
'task': 'mckee_tasks.rate_spread',
'schedule': crontab(minute='*/5', hour='*', day_of_week='*'),
},
'check-forex-rate': {
'task': 'mckee_tasks.fx_rate',
'schedule': crontab(minute='01', hour='*', day_of_week='*'),
},
}
@mckee_app.task(name='mckee_tasks.fx_rate')
def fx_rate():
"""
Keeps the global xe rate variable up to date for the calc_spread task to use
Uses the fixer.io free tier forex API which only has EUR as the base pair so we need to do some division
:return: 31.019741608682857
'backend': 'celery_once.backends.Redis',
'settings': {
'url': env('REDIS'),
'default_timeout': 10 * 60,
'blocking': CELERY_ALWAYS_EAGER,
},
}
BROKER_URL = env('CELERY_BACKEND')
CELERY_ROUTES = {}
CELERYBEAT_SCHEDULE = {
'notify_moderator_by_email': {
'task': 'works.tasks.notify_moderator_by_email',
'schedule': crontab(hour='*/1'),
},
}
# Password validation
# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
app = Celery('neksflis')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
app.conf.beat_schedule = {
'deactivate-unsubscribed-users': {
'task': 'neksflis.account.tasks.deactivate_unsubscribed_users',
'schedule': crontab(minute=0, hour=0),
'args': (),
},
'charge-subscriptions': {
'task': 'neksflis.subscription.tasks.charge_subscriptions',
'schedule': crontab(minute=0, hour='*/6'),
'args': (),
},
'create-new-period-items': {
'task': 'neksflis.subscription.tasks.create_new_period_items',
'schedule': crontab(minute=0, hour='*/3'),
'args': (),
},
'cancel-unpaid-subscriptions': {
'task': 'neksflis.subscription.tasks.cancel_unpaid_subscriptions',
'schedule': crontab(minute=0, hour='*/6'),
CELERY_QUEUES = {
'default': {
"exchange": "default",
"binding_key": "default",
},
'log_loader_queue': {
'exchange': 'log_loader_queue',
'routing_key': 'log_loader_queue',
},
}
CELERYBEAT_SCHEDULE = {
# crontab(hour=0, minute=0, day_of_week='saturday')
'CH_get_stat':{ # example: 'file-backup'
'task': 'api.tasks.task_log_loader_main', # example: 'files.tasks.cleanup'
'schedule': crontab(minute='*/3'),
#'args': (),
'options': {'queue': 'log_loader_queue'},
},'ad_stat_loader':{
'task':'api.tasks.task_adstat_loader',
'schedule':crontab(minute='0',hour='2'),
'options': {'queue': 'log_loader_queue'}},
'CH_get_stat_test':{ # example: 'file-backup'
'task': 'api.tasks_test.task_log_loader_main_test', # example: 'files.tasks.cleanup'
'schedule': crontab(minute='*/3'),
#'args': (),
'options': {'queue': 'log_loader_queue'},
},'ad_stat_loader_test':{
'task':'api.tasks_test.task_adstat_loader_test',
'schedule':crontab(minute='0',hour='*/3'),
'options': {'queue': 'log_loader_queue'}},
# 'ENGINE': 'django.db.backends.postgresql_psycopg2',
# 'NAME': 'students',
# 'USER': '******',
# 'PASSWORD': '******',
# 'HOST': 'localhost',
# 'PORT': '5432',
# },
}
CELERY_BROKER_URL = 'amqp://localhost'
from celery.schedules import crontab
CELERY_BEAT_SCHEDULE = {
'beat': {
'task': 'students.tasks.beat',
'schedule': crontab(minute='*/1'),
}
}
# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME':
'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME':
'django.contrib.auth.password_validation.MinimumLengthValidator',
},
# API Documentation configuration
SWAGGER_SETTINGS = {
'DEFAULT_AUTO_SCHEMA_CLASS': 'vng.utils.schema.CompoundTagsSchema',
}
# User registration settings
ACCOUNT_ACTIVATION_DAYS = 7
REGISTRATION_FORM = 'vng.utils.forms.RegistrationCaptcha'
SILENCED_SYSTEM_CHECKS = ['captcha.recaptcha_test_key_error']
RECAPTCHA_PUBLIC_KEY = '6LfwbaoUAAAAAJ7Bl5o-7pe9DKluPOLX-URNB821'
RECAPTCHA_PRIVATE_KEY = os.getenv('RECAPTCHA_PRIVATE_KEY', 'default')
CELERY_BEAT_SCHEDULE = {
'task-number-one': {
'task': 'vng.testsession.task.purge_sessions',
'schedule': crontab(hour=0, minute=0),
},
'scheduled-test-provider': {
'task': 'vng.servervalidation.task.execute_test_scheduled',
'schedule': crontab(hour=0, minute=0),
},
}
# Elastic APM
ELASTIC_APM = {
'SERVICE_NAME': 'VNG API-Testplatform',
'SECRET_TOKEN': os.getenv('ELASTIC_APM_SECRET_TOKEN', 'default'),
'SERVER_URL': os.getenv('ELASTIC_APM_SERVER_URL', 'http://example.com'),
}
import os
from celery import Celery
from celery.schedules import crontab
from datetime import timedelta
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
app = Celery('our_book')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r'.format(self.request))
app.conf.update(
CELERY_TIMEZONE='Asia/Seoul',
CELERYBEAT_SCHEDULE={
'send_email_overdue_notification': {
'task': 'accounts.tasks.send_email_overdue_notification',
'schedule': crontab(minute=0, hour=11),
'args': ()
},
'send_email_duedate_notification': {
'task': 'accounts.tasks.send_email_return_date_notification',
'schedule': crontab(minute=0, hour=15),
'args': ()
}
}
)
},
# NOTE: pipeline must run on the same worker node as `db`
'pipeline.*': {
'queue': 'pipeline'
},
}
BASE_CONDA_PATH = pathlib.Path('/data/qiime2')
GITHUB_TOKEN = env('GITHUB_TOKEN', default='')
# Don't forget to update local.py when changing here
TASK_TIMES = {
'03_MIN': 60 * 3,
'05_MIN': 60 * 5,
'10_MIN': 60 * 10,
'90_MIN': 60 * 90,
'02_HR': 60 * 60 * 2,
'4A_CRON': crontab(minute=0, hour=4), # daily at 4a
'HRLY_CRON': crontab(minute=23), # hourly
}
def generate_beat_schedule(TASK_TIMES):
return {
'periodic.clean_up_backend': {
'task': 'db.celery_backend_cleanup',
'schedule': TASK_TIMES['4A_CRON'],
},
'periodic.handle_prs': {
'task': 'pipeline.handle_prs',
'schedule': TASK_TIMES['HRLY_CRON'],
},
'periodic.reindex_conda_channels': {
周期性任务还需要启动celery调度命令：python manage.py celerybeat --settings=settings
"""
import datetime
from celery import task
from celery.schedules import crontab
from celery.task import periodic_task
from common.log import logger
import smtplib
from email.mime.text import MIMEText
from email.header import Header
@periodic_task(run_every=crontab(minute='55',
hour='13',
day_of_month='31',
month_of_year='3'))
def time_task():
"""
定义一个 celery 定时任务
"""
now = datetime.datetime.now()
logger.error(u"celery 定时任务执行成功，当前时间：{}".format(now))
time_send_mail()
@periodic_task(run_every=crontab(minute='*/5', hour='*', day_of_week="*"))
def get_time():
"""
celery 周期任务示例
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_serializer
CELERY_RESULT_SERIALIZER = "json"
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-time-limit
# Set to whatever value is adequate in your circumstances
CELERY_TASK_TIME_LIMIT = env.int("CELERY_TASK_TIME_LIMIT",
SECONDS_IN_MINUTE * 30)
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit
# Set to whatever value is adequate in your circumstances
CELERY_TASK_SOFT_TIME_LIMIT = CELERY_TASK_TIME_LIMIT
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
CELERY_BEAT_SCHEDULE = {
"Validate long-term cache integrity every 15 minutes.": {
"task": "sprints.sustainability.tasks.validate_worklog_cache",
"schedule": crontab(minute='*/15'),
"kwargs": {
"long_term": True,
"force_regenerate": False,
},
},
"Recreate long-term cache once per week.": {
"task": "sprints.sustainability.tasks.validate_worklog_cache",
"schedule": crontab(
minute=0,
hour=0,
day_of_week='sun',
),
"kwargs": {
"long_term": True,
"force_regenerate": True,
'ingest:update': {
'task': 'superdesk.io.update_ingest',
# there is internal schedule for updates per provider,
# so this is minimal interval when an update can occur
'schedule': timedelta(seconds=30),
'options': {
'expires': 29
}
},
'ingest:gc': {
'task': 'superdesk.io.gc_ingest',
'schedule': timedelta(minutes=5),
},
'audit:gc': {
'task': 'superdesk.audit.gc_audit',
'schedule': crontab(minute='0', hour=local_to_utc_hour(1))
},
'session:gc': {
'task': 'apps.auth.session_purge',
'schedule': timedelta(minutes=5)
},
'content:gc': {
'task': 'apps.archive.content_expiry',
'schedule': crontab(minute='*/30')
},
'temp_files:gc': {
'task': 'superdesk.commands.temp_file_expiry',
'schedule': crontab(minute='0', hour=local_to_utc_hour(3))
},
'content_api:gc': {
'task': 'content_api.commands.item_expiry',
class CommunityBaseSettings(Settings):
"""Community base settings, don't use this directly."""
# Django settings
SITE_ID = 1
ROOT_URLCONF = 'readthedocs.urls'
SUBDOMAIN_URLCONF = 'readthedocs.core.urls.subdomain'
SINGLE_VERSION_URLCONF = 'readthedocs.core.urls.single_version'
LOGIN_REDIRECT_URL = '/dashboard/'
FORCE_WWW = False
SECRET_KEY = 'replace-this-please' # noqa
ATOMIC_REQUESTS = True
# Debug settings
DEBUG = True
# Domains and URLs
PRODUCTION_DOMAIN = 'readthedocs.org'
PUBLIC_DOMAIN = None
PUBLIC_DOMAIN_USES_HTTPS = False
USE_SUBDOMAIN = False
PUBLIC_API_URL = 'https://{}'.format(PRODUCTION_DOMAIN)
# Some endpoints from the API can be proxied on other domain
# or use the same domain where the docs are being served
# (omit the host if that's the case).
RTD_PROXIED_API_URL = PUBLIC_API_URL
RTD_EXTERNAL_VERSION_DOMAIN = 'external-builds.readthedocs.io'
# Doc Builder Backends
MKDOCS_BACKEND = 'readthedocs.doc_builder.backends.mkdocs'
SPHINX_BACKEND = 'readthedocs.doc_builder.backends.sphinx'
# slumber settings
SLUMBER_API_HOST = 'https://readthedocs.org'
SLUMBER_USERNAME = None
SLUMBER_PASSWORD = None
# Email
DEFAULT_FROM_EMAIL = '*****@*****.**'
SERVER_EMAIL = DEFAULT_FROM_EMAIL
SUPPORT_EMAIL = None
# Sessions
SESSION_COOKIE_DOMAIN = 'readthedocs.org'
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_AGE = 30 * 24 * 60 * 60 # 30 days
SESSION_SAVE_EVERY_REQUEST = True
# This cookie is used in cross-origin API requests from *.readthedocs.io to readthedocs.org
SESSION_COOKIE_SAMESITE = None
# CSRF
CSRF_COOKIE_HTTPONLY = True
CSRF_COOKIE_AGE = 30 * 24 * 60 * 60
# Security & X-Frame-Options Middleware
# https://docs.djangoproject.com/en/1.11/ref/middleware/#django.middleware.security.SecurityMiddleware
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
X_FRAME_OPTIONS = 'DENY'
# Content Security Policy
# https://django-csp.readthedocs.io/
CSP_BLOCK_ALL_MIXED_CONTENT = True
CSP_DEFAULT_SRC = None # This could be improved
CSP_FRAME_ANCESTORS = ("'none'",)
CSP_OBJECT_SRC = ("'none'",)
CSP_REPORT_URI = None
CSP_REPORT_ONLY = True # Set to false to enable CSP in blocking mode
CSP_EXCLUDE_URL_PREFIXES = (
"/admin/",
)
# Read the Docs
READ_THE_DOCS_EXTENSIONS = ext
RTD_LATEST = 'latest'
RTD_LATEST_VERBOSE_NAME = 'latest'
RTD_STABLE = 'stable'
RTD_STABLE_VERBOSE_NAME = 'stable'
RTD_CLEAN_AFTER_BUILD = False
# Database and API hitting settings
DONT_HIT_API = False
DONT_HIT_DB = True
SYNC_USER = getpass.getuser()
USER_MATURITY_DAYS = 7
# override classes
CLASS_OVERRIDES = {}
DOC_PATH_PREFIX = '_/'
# Application classes
@property
def INSTALLED_APPS(self): # noqa
apps = [
'django.contrib.auth',
'django.contrib.admin',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.staticfiles',
'django.contrib.messages',
'django.contrib.humanize',
# third party apps
'dj_pagination',
'taggit',
'django_gravatar',
'rest_framework',
'rest_framework.authtoken',
'corsheaders',
'textclassifier',
'annoying',
'django_extensions',
'crispy_forms',
'messages_extends',
'django_elasticsearch_dsl',
'django_filters',
'polymorphic',
# our apps
'readthedocs.projects',
'readthedocs.builds',
'readthedocs.core',
'readthedocs.doc_builder',
'readthedocs.oauth',
'readthedocs.redirects',
'readthedocs.rtd_tests',
'readthedocs.api.v2',
'readthedocs.api.v3',
'readthedocs.gold',
'readthedocs.payments',
'readthedocs.notifications',
'readthedocs.integrations',
'readthedocs.analytics',
'readthedocs.sphinx_domains',
'readthedocs.search',
# allauth
'allauth',
'allauth.account',
'allauth.socialaccount',
'allauth.socialaccount.providers.github',
'allauth.socialaccount.providers.gitlab',
'allauth.socialaccount.providers.bitbucket',
'allauth.socialaccount.providers.bitbucket_oauth2',
]
if ext:
apps.append('django_countries')
apps.append('readthedocsext.donate')
apps.append('readthedocsext.embed')
apps.append('readthedocsext.spamfighting')
return apps
@property
def USE_PROMOS(self): # noqa
return 'readthedocsext.donate' in self.INSTALLED_APPS
MIDDLEWARE = (
'readthedocs.core.middleware.ReadTheDocsSessionMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'dj_pagination.middleware.PaginationMiddleware',
'readthedocs.core.middleware.SubdomainMiddleware',
'readthedocs.core.middleware.SingleVersionMiddleware',
'corsheaders.middleware.CorsMiddleware',
'csp.middleware.CSPMiddleware',
)
AUTHENTICATION_BACKENDS = (
# Needed to login by username in Django admin, regardless of `allauth`
'django.contrib.auth.backends.ModelBackend',
# `allauth` specific authentication methods, such as login by e-mail
'allauth.account.auth_backends.AuthenticationBackend',
)
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
'OPTIONS': {
'min_length': 9,
}
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
MESSAGE_STORAGE = 'readthedocs.notifications.storages.FallbackUniqueStorage'
NOTIFICATION_BACKENDS = [
'readthedocs.notifications.backends.EmailBackend',
'readthedocs.notifications.backends.SiteBackend',
]
# Paths
SITE_ROOT = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
TEMPLATE_ROOT = os.path.join(SITE_ROOT, 'readthedocs', 'templates')
DOCROOT = os.path.join(SITE_ROOT, 'user_builds')
UPLOAD_ROOT = os.path.join(SITE_ROOT, 'user_uploads')
CNAME_ROOT = os.path.join(SITE_ROOT, 'cnames')
LOGS_ROOT = os.path.join(SITE_ROOT, 'logs')
PRODUCTION_ROOT = os.path.join(SITE_ROOT, 'prod_artifacts')
PRODUCTION_MEDIA_ARTIFACTS = os.path.join(PRODUCTION_ROOT, 'media')
# Assets and media
STATIC_ROOT = os.path.join(SITE_ROOT, 'static')
STATIC_URL = '/static/'
MEDIA_ROOT = os.path.join(SITE_ROOT, 'media/')
MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/media/admin/'
STATICFILES_DIRS = [
os.path.join(SITE_ROOT, 'readthedocs', 'static'),
os.path.join(SITE_ROOT, 'media'),
]
STATICFILES_FINDERS = [
'readthedocs.core.static.SelectiveFileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
]
PYTHON_MEDIA = False
# Django Storage subclass used to write build artifacts to cloud or local storage
# https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [TEMPLATE_ROOT],
'OPTIONS': {
'debug': DEBUG,
'context_processors': [
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media',
'django.template.context_processors.request',
# Read the Docs processor
'readthedocs.core.context_processors.readthedocs_processor',
],
'loaders': [
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
],
},
},
]
# Cache
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'PREFIX': 'docs',
}
}
CACHE_MIDDLEWARE_SECONDS = 60
# I18n
TIME_ZONE = 'UTC'
USE_TZ = True
LANGUAGE_CODE = 'en-us'
LANGUAGES = (
('ca', gettext('Catalan')),
('en', gettext('English')),
('es', gettext('Spanish')),
('pt-br', gettext('Brazilian Portuguese')),
('nb', gettext('Norwegian Bokmål')),
('fr', gettext('French')),
('ru', gettext('Russian')),
('de', gettext('German')),
('gl', gettext('Galician')),
('vi', gettext('Vietnamese')),
('zh-cn', gettext('Simplified Chinese')),
('zh-tw', gettext('Traditional Chinese')),
('ja', gettext('Japanese')),
('uk', gettext('Ukrainian')),
('it', gettext('Italian')),
('ko', gettext('Korean')),
)
LOCALE_PATHS = [
os.path.join(SITE_ROOT, 'readthedocs', 'locale'),
]
USE_I18N = True
USE_L10N = True
# Celery
CELERY_APP_NAME = 'readthedocs'
CELERY_ALWAYS_EAGER = True
CELERYD_TASK_TIME_LIMIT = 60 * 60 # 60 minutes
CELERY_SEND_TASK_ERROR_EMAILS = False
CELERYD_HIJACK_ROOT_LOGGER = False
# This stops us from pre-fetching a task that then sits around on the builder
CELERY_ACKS_LATE = True
# Don't queue a bunch of tasks in the workers
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_CREATE_MISSING_QUEUES = True
CELERY_DEFAULT_QUEUE = 'celery'
CELERYBEAT_SCHEDULE = {
# Ran every hour on minute 30
'hourly-remove-orphan-symlinks': {
'task': 'readthedocs.projects.tasks.broadcast_remove_orphan_symlinks',
'schedule': crontab(minute=30),
'options': {'queue': 'web'},
},
'quarter-finish-inactive-builds': {
'task': 'readthedocs.projects.tasks.finish_inactive_builds',
'schedule': crontab(minute='*/15'),
'options': {'queue': 'web'},
},
'every-three-hour-clear-persistent-messages': {
'task': 'readthedocs.core.tasks.clear_persistent_messages',
'schedule': crontab(minute=0, hour='*/3'),
'options': {'queue': 'web'},
},
'every-day-delete-old-search-queries': {
'task': 'readthedocs.search.tasks.delete_old_search_queries_from_db',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': 'web'},
}
}
MULTIPLE_APP_SERVERS = [CELERY_DEFAULT_QUEUE]
MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]
# Sentry
SENTRY_CELERY_IGNORE_EXPECTED = True
# Docker
DOCKER_ENABLE = False
DOCKER_SOCKET = 'unix:///var/run/docker.sock'
# This settings has been deprecated in favor of DOCKER_IMAGE_SETTINGS
DOCKER_BUILD_IMAGES = None
DOCKER_LIMITS = {'memory': '200m', 'time': 600}
# User used to create the container.
# In production we use the same user than the one defined by the
# ``USER docs`` instruction inside the Dockerfile.
# In development, we can use the "UID:GID" of the current user running the
# instance to avoid file permissions issues.
# https://docs.docker.com/engine/reference/run/#user
RTD_DOCKER_USER = '******'
RTD_DOCKER_COMPOSE = False
DOCKER_DEFAULT_IMAGE = 'readthedocs/build'
DOCKER_VERSION = 'auto'
DOCKER_DEFAULT_VERSION = 'latest'
DOCKER_IMAGE = '{}:{}'.format(DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION)
DOCKER_IMAGE_SETTINGS = {
# A large number of users still have this pinned in their config file.
# We must have documented it at some point.
'readthedocs/build:2.0': {
'python': {
'supported_versions': [2, 2.7, 3, 3.5],
'default_version': {
2: 2.7,
3: 3.5,
},
},
},
'readthedocs/build:4.0': {
'python': {
'supported_versions': [2, 2.7, 3, 3.5, 3.6, 3.7],
'default_version': {
2: 2.7,
3: 3.7,
},
},
},
'readthedocs/build:5.0': {
'python': {
'supported_versions': [2, 2.7, 3, 3.5, 3.6, 3.7, 'pypy3.5'],
'default_version': {
2: 2.7,
3: 3.7,
},
},
},
'readthedocs/build:6.0': {
'python': {
'supported_versions': [2, 2.7, 3, 3.5, 3.6, 3.7, 3.8, 'pypy3.5'],
'default_version': {
2: 2.7,
3: 3.7,
},
},
},
'readthedocs/build:7.0': {
'python': {
'supported_versions': [2, 2.7, 3, 3.5, 3.6, 3.7, 3.8, 'pypy3.5'],
'default_version': {
2: 2.7,
3: 3.7,
},
},
},
}
# Alias tagged via ``docker tag`` on the build servers
DOCKER_IMAGE_SETTINGS.update({
'readthedocs/build:stable': DOCKER_IMAGE_SETTINGS.get('readthedocs/build:5.0'),
'readthedocs/build:latest': DOCKER_IMAGE_SETTINGS.get('readthedocs/build:6.0'),
'readthedocs/build:testing': DOCKER_IMAGE_SETTINGS.get('readthedocs/build:7.0'),
})
# All auth
ACCOUNT_ADAPTER = 'readthedocs.core.adapters.AccountAdapter'
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = 'mandatory'
ACCOUNT_AUTHENTICATION_METHOD = 'username_email'
ACCOUNT_ACTIVATION_DAYS = 7
SOCIALACCOUNT_AUTO_SIGNUP = False
SOCIALACCOUNT_PROVIDERS = {
'github': {
'SCOPE': [
'user:email',
'read:org',
'admin:repo_hook',
'repo:status',
],
},
'gitlab': {
'SCOPE': [
'api',
'read_user',
],
},
}
# CORS
CORS_ORIGIN_REGEX_WHITELIST = (
r'^http://(.+)\.readthedocs\.io$',
r'^https://(.+)\.readthedocs\.io$',
)
# So people can post to their accounts
CORS_ALLOW_CREDENTIALS = True
CORS_ALLOW_HEADERS = (
'x-requested-with',
'content-type',
'accept',
'origin',
'authorization',
'x-csrftoken'
)
# RTD Settings
REPO_LOCK_SECONDS = 30
ALLOW_PRIVATE_REPOS = False
DEFAULT_PRIVACY_LEVEL = 'public'
DEFAULT_VERSION_PRIVACY_LEVEL = 'public'
GROK_API_HOST = 'https://api.grokthedocs.com'
SERVE_DOCS = ['public']
ALLOW_ADMIN = True
# Elasticsearch settings.
ES_HOSTS = ['search:9200']
ELASTICSEARCH_DSL = {
'default': {
'hosts': 'search:9200'
},
}
# Chunk size for elasticsearch reindex celery tasks
ES_TASK_CHUNK_SIZE = 100
# Info from Honza about this:
# The key to determine shard number is actually usually not the node count,
# but the size of your data.
# There are advantages to just having a single shard in an index since
# you don't have to do the distribute/collect steps when executing a search.
# If your data will allow it (not significantly larger than 40GB)
# I would recommend going to a single shard and one replica meaning
# any of the two nodes will be able to serve any search without talking to the other one.
# Scaling to more searches will then just mean adding a third node
# and a second replica resulting in immediate 50% bump in max search throughput.
ES_INDEXES = {
'project': {
'name': 'project_index',
'settings': {'number_of_shards': 1,
'number_of_replicas': 1
}
},
'page': {
'name': 'page_index',
'settings': {
'number_of_shards': 1,
'number_of_replicas': 1,
}
},
}
# ANALYZER = 'analysis': {
# 'analyzer': {
# 'default_icu': {
# 'type': 'custom',
# 'tokenizer': 'icu_tokenizer',
# 'filter': ['word_delimiter', 'icu_folding', 'icu_normalizer'],
# }
# }
# }
# Disable auto refresh for increasing index performance
ELASTICSEARCH_DSL_AUTO_REFRESH = False
ALLOWED_HOSTS = ['*']
ABSOLUTE_URL_OVERRIDES = {
'auth.user': lambda o: '/profiles/{}/'.format(o.username)
}
INTERNAL_IPS = ('127.0.0.1',)
# Taggit
# https://django-taggit.readthedocs.io
TAGGIT_TAGS_FROM_STRING = 'readthedocs.projects.tag_utils.rtd_parse_tags'
# Stripe
STRIPE_SECRET = None
STRIPE_PUBLISHABLE = None
# Do Not Track support
DO_NOT_TRACK_ENABLED = False
# Misc application settings
GLOBAL_ANALYTICS_CODE = None
DASHBOARD_ANALYTICS_CODE = None # For the dashboard, not docs
GRAVATAR_DEFAULT_IMAGE = 'https://assets.readthedocs.org/static/images/silhouette.png' # NOQA
OAUTH_AVATAR_USER_DEFAULT_URL = GRAVATAR_DEFAULT_IMAGE
OAUTH_AVATAR_ORG_DEFAULT_URL = GRAVATAR_DEFAULT_IMAGE
RESTRICTEDSESSIONS_AUTHED_ONLY = True
RESTRUCTUREDTEXT_FILTER_SETTINGS = {
'cloak_email_addresses': True,
'file_insertion_enabled': False,
'raw_enabled': False,
'strip_comments': True,
'doctitle_xform': True,
'sectsubtitle_xform': True,
'initial_header_level': 2,
'report_level': 5,
'syntax_highlight': 'none',
'math_output': 'latex',
'field_name_limit': 50,
}
REST_FRAMEWORK = {
'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',),
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', # NOQA
'DEFAULT_THROTTLE_RATES': {
'anon': '5/minute',
'user': '******',
},
'PAGE_SIZE': 10,
'TEST_REQUEST_DEFAULT_FORMAT': 'json',
}
SILENCED_SYSTEM_CHECKS = ['fields.W342']
# Logging
LOG_FORMAT = '%(name)s:%(lineno)s[%(process)d]: %(levelname)s %(message)s'
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'default': {
'format': LOG_FORMAT,
'datefmt': '%d/%b/%Y %H:%M:%S',
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'default'
},
'debug': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': os.path.join(LOGS_ROOT, 'debug.log'),
'formatter': 'default',
},
'null': {
'class': 'logging.NullHandler',
},
},
'loggers': {
'': { # root logger
'handlers': ['debug', 'console'],
# Always send from the root, handlers can filter levels
'level': 'DEBUG',
},
'readthedocs': {
'handlers': ['debug', 'console'],
'level': 'DEBUG',
# Don't double log at the root logger for these.
'propagate': False,
},
'django.security.DisallowedHost': {
'handlers': ['null'],
'propagate': False,
},
},
}
def execute_task():
"""
执行 celery 异步任务
调用celery任务方法:
task.delay(arg1, arg2, kwarg1='x', kwarg2='y')
task.apply_async(args=[arg1, arg2], kwargs={'kwarg1': 'x', 'kwarg2': 'y'})
delay(): 简便方法，类似调用普通函数
apply_async(): 设置celery的额外执行选项时必须使用该方法，如定时（eta）等
详见 ：http://celery.readthedocs.org/en/latest/userguide/calling.html
"""
now = datetime.datetime.now()
logger.error(u"celery 定时任务启动，将在60s后执行，当前时间：{}".format(now))
# 调用定时任务
async_task.apply_async(args=[now.hour, now.minute],
eta=now + datetime.timedelta(seconds=60))
@periodic_task(run_every=crontab(minute='*/5', hour='*', day_of_week="*"))
def get_time():
"""
celery 周期任务示例
run_every=crontab(minute='*/5', hour='*', day_of_week="*")：每 5 分钟执行一次任务
periodic_task：程序运行时自动触发周期任务
"""
execute_task()
now = datetime.datetime.now()
logger.error(u"celery 周期任务调用成功，当前时间：{}".format(now))
#!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2014 Mozilla Corporation # # Contributors: # Anthony Verez [email protected] from celery.schedules import crontab import time import logging ALERTS = { 'bro_intel.AlertBroIntel': crontab(minute='*/1'), 'bro_notice.AlertBroNotice': crontab(minute='*/1'), 'bruteforce_ssh.AlertBruteforceSsh': crontab(minute='*/1'), 'cloudtrail.AlertCloudtrail': crontab(minute='*/1'), 'fail2ban.AlertFail2ban': crontab(minute='*/1'), } RABBITMQ = { 'mqserver': 'localhost', 'mquser': '******', 'mqpassword': '******', 'mqport': 5672, 'alertexchange': 'alerts', 'alertqueue': 'mozdef.alert' }
)
else:
logger.error(
'Send fax error, will retry: %s',
exc,
exc_info=sys.exc_info(),
)
send_fax.retry(
countdown=300,
args=[comm_id, subject, body, error_count],
kwargs=kwargs,
exc=exc,
)
@periodic_task(run_every=crontab(hour=5, minute=0),
time_limit=10 * 60,
soft_time_limit=570,
name='muckrock.foia.tasks.followup_requests')
def followup_requests():
"""Follow up on any requests that need following up on"""
log = []
# weekday returns 5 for sat and 6 for sun
is_weekday = date.today().weekday() < 5
if (config.ENABLE_FOLLOWUP
and (config.ENABLE_WEEKEND_FOLLOWUP or is_weekday)):
try:
num_requests = FOIARequest.objects.get_followup().count()
for foia in FOIARequest.objects.get_followup():
try:
foia.followup()
def schedule(self):
return schedules.crontab(minute=self.minute,
hour=self.hour,
day_of_week=self.day_of_week,
day_of_month=self.day_of_month,
month_of_year=self.month_of_year)
class UipaOrgThemeBase(ThemeBase):
FROIDE_THEME = 'uipa_org.theme'
SITE_NAME = "UIPA.org"
SITE_EMAIL = "*****@*****.**"
SITE_URL = 'http://localhost:8000'
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
STATIC_ROOT = os.path.abspath(os.path.join(PROJECT_ROOT, "..", "public"))
FIXTURE_DIRS = ('fixtures',)
MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
SECRET_KEY = os_env('SECRET_KEY')
MEDIA_ROOT = os_env('MEDIA_ROOT')
DATA_UPLOAD_MAX_MEMORY_SIZE = 26214400 # 25MB
TAGGING_AUTOCOMPLETE_MAX_TAGS = 100
@property
def INSTALLED_APPS(self):
installed = super(UipaOrgThemeBase, self).INSTALLED_APPS
installed += [
'celery_haystack',
'djcelery_email',
'django.contrib.redirects',
'uipa_org.uipa_constants',
'uipa_org.theme.templatetags.uipa_extras',
'tinymce',
'raven.contrib.django.raven_compat'
]
return installed
MIDDLEWARE_CLASSES = [
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.common.CommonMiddleware',
'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
'froide.account.middleware.AcceptNewTermsMiddleware',
]
TINYMCE_DEFAULT_CONFIG = {
'plugins': "table,spellchecker,paste,searchreplace",
'theme': "advanced",
'cleanup_on_startup': False
}
SECRET_URLS = values.DictValue({
"admin": "uipa-admin",
"postmark_inbound": "uipa_postmark_inbound",
"postmark_bounce": "uipa_postmark_bounce"
})
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': 'http://127.0.0.1:9200/',
'INDEX_NAME': 'haystack',
}
}
TIME_ZONE = values.Value('Pacific/Honolulu')
CELERY_IMPORTS = ('uipa_org.tasks',)
CELERY_TIMEZONE = values.Value('Pacific/Honolulu')
CELERYBEAT_SCHEDULE = {
'fetch-mail': {
'task': 'froide.foirequest.tasks.fetch_mail',
'schedule': crontab(),
},
'detect-asleep': {
'task': 'froide.foirequest.tasks.detect_asleep',
'schedule': crontab(hour=0, minute=0),
},
'detect-overdue': {
'task': 'froide.foirequest.tasks.detect_overdue',
'schedule': crontab(hour=0, minute=0),
},
'update-foirequestfollowers': {
'task': 'froide.foirequestfollower.tasks.batch_update',
'schedule': crontab(hour=0, minute=0),
},
'classification-reminder': {
'task': 'froide.foirequest.tasks.classification_reminder',
'schedule': crontab(hour=7, minute=0, day_of_week=6),
},
'uipa-private_public_reminder': {
'task': 'uipa_org.tasks.private_public_reminder',
'schedule': crontab(hour=0, minute=0),
},
'uipa-make_public_private': {
'task': 'uipa_org.tasks.make_private_public',
'schedule': crontab(hour=0, minute=0),
},
'uipa-deferred_message_notification': {
'task': 'uipa_org.tasks.deferred_message_notification',
'schedule': crontab(hour=6, minute=0),
},
}
CELERY_RESULT_BACKEND = 'rpc'
CELERY_RESULT_PERSISTENT = True
@property
def FROIDE_CONFIG(self):
config = super(UipaOrgThemeBase, self).FROIDE_CONFIG
config.update(dict(
currency="Dollars",
create_new_publicbody=False,
publicbody_empty=False,
user_can_hide_web=True,
public_body_officials_public=True,
public_body_officials_email_public=False,
request_public_after_due_days=14,
payment_possible=False,
default_law=1,
greetings=[rec(u"Aloha (?:Mr\.?|Ms\.? .*?)")],
closings=[rec(u"Mahalo,?")],
public_body_boosts={},
dryrun=True,
dryrun_domain="beta.uipa.org",
allow_pseudonym=False,
# doc_conversion_binary=None, # replace with libreoffice instance
doc_conversion_binary="/Applications/LibreOffice.app/Contents/MacOS/soffice",
doc_conversion_call_func=None, # see settings_test for use
api_activated=True,
search_engine_query='http://www.google.com/search?as_q=%(query)s&as_epq=&as_oq=&as_eq=&hl=en&lr=&cr=&as_ft=i&as_filetype=&as_qdr=all&as_occt=any&as_dt=i&as_sitesearch=%(domain)s&as_rights=&safe=images',
show_public_body_employee_name=False,
make_public_num_days_after_due_date=365,
ga_tracking_id=os_env('GA_TRACKING_ID'),
))
return config
def configure_celery(flask_app, celery, test_config=None):
database_url = shared_config["db"]["url"]
engine_args_literal = ast.literal_eval(
shared_config["db"]["engine_args_literal"])
redis_url = shared_config["redis"]["url"]
if test_config is not None:
if "db" in test_config:
if "url" in test_config["db"]:
database_url = test_config["db"]["url"]
# Update celery configuration
celery.conf.update(
imports=[
"src.tasks.index", "src.tasks.index_blacklist",
"src.tasks.index_cache", "src.tasks.index_plays",
"src.tasks.index_metrics"
],
beat_schedule={
"update_discovery_provider": {
"task": "update_discovery_provider",
"schedule": timedelta(seconds=5),
},
"update_ipld_blacklist": {
"task": "update_ipld_blacklist",
"schedule": timedelta(seconds=60),
},
"update_cache": {
"task": "update_discovery_cache",
"schedule": timedelta(seconds=60)
},
"update_play_count": {
"task": "update_play_count",
"schedule": timedelta(seconds=5)
},
"update_metrics": {
"task": "update_metrics",
"schedule": crontab(minute=0, hour="*")
}
},
task_serializer="json",
accept_content=["json"],
broker_url=redis_url,
)
# Initialize DB object for celery task context
db = SessionManager(database_url, engine_args_literal)
logger.info('Database instance initialized!')
# Initialize IPFS client for celery task context
gateway_addrs = shared_config["ipfs"]["gateway_hosts"].split(',')
gateway_addrs.append(
shared_config["discprov"]["user_metadata_service_url"])
logger.warning(f"__init__.py | {gateway_addrs}")
ipfs_client = IPFSClient(shared_config["ipfs"]["host"],
shared_config["ipfs"]["port"], gateway_addrs)
# Initialize Redis connection
redis_inst = redis.Redis.from_url(url=redis_url)
# Clear existing lock if present
redis_inst.delete("disc_prov_lock")
logger.info('Redis instance initialized!')
# Initialize custom task context with database object
class DatabaseTask(Task):
def __init__(self, *args, **kwargs):
self._db = db
self._web3_provider = web3
self._abi_values = abi_values
self._shared_config = shared_config
self._ipfs_client = ipfs_client
self._redis = redis_inst
@property
def abi_values(self):
return self._abi_values
@property
def web3(self):
return self._web3_provider
@property
def db(self):
return self._db
@property
def shared_config(self):
return self._shared_config
@property
def ipfs_client(self):
return self._ipfs_client
@property
def redis(self):
return self._redis
celery.autodiscover_tasks(["src.tasks"], "index", True)
# Subclassing celery task with discovery provider context
# Provided through properties defined in 'DatabaseTask'
celery.Task = DatabaseTask
celery.finalize()
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from celery.schedules import crontab
from indico.core.celery import celery
from indico.core.db import db
from indico_livesync.models.agents import LiveSyncAgent
from indico_livesync.util import clean_old_entries
@celery.periodic_task(run_every=crontab(minute='*/15'), plugin='livesync')
def scheduled_update():
from indico_livesync.plugin import LiveSyncPlugin
clean_old_entries()
for agent in LiveSyncAgent.find_all():
if agent.backend is None:
LiveSyncPlugin.logger.warning(
'Skipping agent %s; backend not found', agent.name)
continue
if not agent.initial_data_exported:
LiveSyncPlugin.logger.warning(
'Skipping agent %s; initial export not performed yet',
agent.name)
continue
LiveSyncPlugin.logger.info('Running agent %s', agent.name)
agent.create_backend().run()
CELERY_BEAT_SCHEDULE = {
"delete-empty-allocations": {
"task": "saleor.warehouse.tasks.delete_empty_allocations_task",
"schedule": timedelta(days=1),
},
"deactivate-preorder-for-variants": {
"task": "saleor.product.tasks.deactivate_preorder_for_variants_task",
"schedule": timedelta(hours=1),
},
"delete-expired-reservations": {
"task": "saleor.warehouse.tasks.delete_expired_reservations_task",
"schedule": timedelta(days=1),
},
"delete-expired-checkouts": {
"task": "saleor.checkout.tasks.delete_expired_checkouts",
"schedule": crontab(hour=0, minute=0),
},
"delete-outdated-event-data": {
"task": "saleor.core.tasks.delete_event_payloads_task",
"schedule": timedelta(days=1),
},
"deactivate-expired-gift-cards": {
"task": "saleor.giftcard.tasks.deactivate_expired_cards_task",
"schedule": crontab(hour=0, minute=0),
},
"update-stocks-quantity-allocated": {
"task": "saleor.warehouse.tasks.update_stocks_quantity_allocated_task",
"schedule": crontab(hour=0, minute=0),
},
"delete-old-export-files": {
"task": "saleor.csv.tasks.delete_old_export_files",
def register_periodical_tasks(sender, **kwargs) -> None:
sender.add_periodic_task(
crontab(hour='*/24', minute=0),
update_database.s(),
)
from celery.task import periodic_task, task
from celery.utils.log import get_task_logger
from corehq.apps.domain.calculations import CALC_FNS, _all_domain_stats
from corehq.apps.hqadmin.escheck import check_cluster_health, check_case_index, CLUSTER_HEALTH, check_xform_index, check_exchange_index
from corehq.apps.reports.models import (ReportNotification,
UnsupportedScheduledReportError,
HQGroupExportConfiguration)
from corehq.elastic import get_es
from corehq.pillows.mappings.domain_mapping import DOMAIN_INDEX
from couchexport.groupexports import export_for_group
from dimagi.utils.logging import notify_exception
logging = get_task_logger(__name__)
@periodic_task(run_every=crontab(hour=[8, 14], minute="0", day_of_week="*"))
def check_es_index():
"""
Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch
This code is also called in the HQ admin page as well
"""
es_status = {}
es_status.update(check_cluster_health())
es_status.update(check_case_index())
es_status.update(check_xform_index())
es_status.update(check_exchange_index())
do_notify = False
message = []
from __future__ import absolute_import, unicode_literals
from celery import Celery
from celery.schedules import crontab
app = Celery('Pipeline',
broker='amqp://',
backend='amqp://',
include=['Pipeline.tasks'])
app.conf.timezone = 'US/Eastern'
app.conf.beat_schedule = {
# Execute every two minutes
'run-every-2-minute': {
'task': 'tasks.run',
'schedule': crontab(minute='*/2')
}
# Executes every Day morning at 8:30 a.m.
# 'run-every-day-morning': {
# 'task': 'tasks.run',
# # 'schedule': crontab(hour=8, minute=30),
# },
# 'add-every-10-seconds': {
# 'task': 'tasks.add',
# 'schedule': 10.0,
# 'args': (16, 16)
# },
# 'add-every-1-minute': {
