def Reset(fingerprint=None, covername=None):
if not thirtythirty.hdd.drives_are_unlocked():
logger.warning('Drives not unlocked yet.')
Sched = django_rq.get_scheduler('default')
Sched.enqueue_in(datetime.timedelta(minutes=30), Reset, fingerprint=fingerprint)
exit()
if not exists(TTS.PASSPHRASE_CACHE):
logger.warning('I need a passphrase to continue - rescheduling.')
Sched = django_rq.get_scheduler()
Sched.enqueue_in(datetime.timedelta(minutes=30), Reset, fingerprint=fingerprint)
exit()
passphrase = file(TTS.PASSPHRASE_CACHE, 'r').read()
Addr = addressbook.address.objects.filter(
Q(fingerprint=fingerprint) |\
Q(covername=covername))
if len(Addr) != 1:
logger.critical('Got an address reset request with multiple matches: %s' % Addr)
exit(-1)
logger.debug('Request for %s state reset granted' % Addr[0])
Addr[0].delete_local_state(passphrase)
def ready(self):
from thefederation.social import make_daily_post
from thefederation.tasks import aggregate_daily_stats
from thefederation.tasks import poll_nodes
scheduler = django_rq.get_scheduler('high')
# Delete any existing jobs in the scheduler when the app starts up
for job in scheduler.get_jobs():
job.delete()
scheduler.schedule(
scheduled_time=datetime.datetime.utcnow(),
func=aggregate_daily_stats,
interval=5500,
)
scheduler.cron(
'0 10 * * *',
func=make_daily_post,
queue_name='high',
)
scheduler = django_rq.get_scheduler('medium')
scheduler.schedule(
scheduled_time=datetime.datetime.utcnow(),
func=poll_nodes,
interval=10800,
)
def Pull(covername=None, fingerprint=None, email=None):
logger.debug('Keyserver pull')
if not exists(TTS.PASSPHRASE_CACHE):
logger.warning('I need a passphrase to continue - rescheduling.')
Sched = django_rq.get_scheduler()
Sched.enqueue_in(datetime.timedelta(minutes=30), Pull, covername=covername)
exit()
passphrase = file(TTS.PASSPHRASE_CACHE, 'r').read()
try: Ratchet_Objects.decrypt_database(passphrase)
except thirtythirty.exception.Target_Exists: pass
Me = addressbook.utils.my_address()
if email and not covername:
Email_Strip = re.sub('@.*', '', email.upper())
DeJunk = re.sub('[^ .A-Z]+', '', Email_Strip)
covername = re.sub('[\. ]+', ' ', DeJunk)
try:
Resp = addressbook.gpg.pull_from_keyserver(covername=covername,
fingerprint=fingerprint)
if Resp:
return Resp.fingerprint
else:
return None
except addressbook.exception.MultipleMatches as Matches:
logger.debug(Matches)
emailclient.utils.submit_to_smtpd(Payload="""The upstream server returned many keys for "%s".
This is exciting new territory in errors! Please fill out a bug report! Thank you!
""" % covername,
Destination=Me.email,
Subject='FATAL problem - many users go by "%s"' % covername,
From='Sysop <root>')
except addressbook.exception.NoKeyMatch:
logger.debug('No matches')
emailclient.utils.submit_to_smtpd(Payload="""No upstream server knows "%s".
This may be a typo in the covername - please bother a developer to write the synonym code in a bug report! Thank you!
""" % covername,
Destination=Me.email,
Subject='FATAL problem - no one by the name of "%s"' % covername,
From='Sysop <root>')
except addressbook.exception.KeyserverTimeout:
logger.debug('Keyservers all timed out - TEMPFAIL')
emailclient.utils.submit_to_smtpd(Payload="""The upstream server seems to have experienced a temporary problem during key lookup for "%s".
We'll try again in a bit and see if it magically starts working.
""" % covername,
Destination=Me.email,
Subject='Temporary problem - key lookup for "%s"' % covername,
From='Sysop <root>')
Sched = django_rq.get_scheduler('default')
TEMPFAIL = Sched.enqueue_in(datetime.timedelta(minutes=30), Pull, covername=covername)
return TEMPFAIL.result
def perform_create(self, serializer):
letter = serializer.save()
django_rq.enqueue(mail.send_imidiate_notifications, letter)
till_sending = 24 * 3 * settings.SECONDS_IN_HOUR
sending_datetime = datetime.now() + timedelta(seconds=till_sending)
scheduler = django_rq.get_scheduler()
scheduler.enqueue_at(sending_datetime, mail.send_letter, letter)
def CampaignUnschedule(request, pk):
campaign = get_object_or_404(Campaign, pk=pk)
scheduler = django_rq.get_scheduler('default')
scheduler.cancel(campaign.jobId)
campaign.jobId = None
campaign.save()
return redirect(campaign)
# def TweetActionReturn(request, tweet_id, action):
# tweet = get_object_or_404(Tweet, tweet_id=tweet_id)
#
# tweetAction = tweet.tweet_actions.all()[0]
# if (action == "clicked" and tweetAction.shortened_urls != None):
# bitly = bitly_api.Connection(access_token = settings.BITLY_TOKEN)
# if (bitly.link_clicks(tweetAction.getShortenedUrls()[0]) > 1):
# tweetAction.clicked = True
# elif (action == "created"):
# tweetAction.prefrCreated = True
# tweetAction.save()
#
# return JsonResponse({'status': 'ok'})
def schedule_periodic_index_rebuilds():
base_dt = datetime.datetime.now(tz=get_default_timezone())
replace_time = {
'hour': 1,
'minute': 15,
'second': 0,
}
if base_dt.hour > replace_time['hour']:
base_dt = base_dt + datetime.datetime.timedelta(days=1)
after_midnight = base_dt.replace(**replace_time).astimezone(utc)
scheduler = django_rq.get_scheduler('default')
# Delete existing 'rebuild_indices' jobs
for job in scheduler.get_jobs():
if job.func_name == 'search.tasks.rebuild_indices':
job.delete()
scheduler.schedule(
scheduled_time=after_midnight,
func=rebuild_indices,
args=[search_config._settings],
kwargs={'verbosity': 1},
interval=86400, # Reindex daily
repeat=None,
result_ttl=None
)
def Redeem(self):
if self.status == Gift.GIFT_STATUS_CREATED:
try:
card = Card.Card.objects.filter(vendor=self.product.vendor, user=None, master=False)[:1]
except Card.Card.DoesNotExist:
card = None
expireTimeUTC = None
if card:
card = card[0]
card.user = self.receiver
card.save()
queue = rq.get_queue('high')
load = queue.enqueue(Card.SetBalance, card.id, self.product.price)
scheduler = rq.get_scheduler('low')
expireTimeUTC = datetime.utcnow() + timedelta(minutes=5)
expireTime = datetime.now() + timedelta(minutes=5)
unload = scheduler.enqueue_at(expireTime, CheckRedemption, self.id)
self.activated = datetime.utcnow().replace(tzinfo=utc)
self.status = self.GIFT_STATUS_ACTIVE
self.payment = card
self.save()
elif self.status == Gift.GIFT_STATUS_ACTIVE:
card = self.payment
expireTimeUTC = self.activated + timedelta(minutes=5)
return (card, expireTimeUTC)
def _select_run_method(context, interactive, function, after):
"""Return a function that either executes the task directly (if
`interactive` is True), enqueues it right away or schedules its enqueueing
(if `after` is given).
"""
if interactive:
return function
set_queue(context)
if after:
# FIXME: what about timeout= and result_ttl= for scheduled tasks?
scheduler = django_rq.get_scheduler(context['queue'], )
if isinstance(after, timedelta):
enqueue = scheduler.enqueue_in
elif isinstance(after, datetime):
enqueue = scheduler.enqueue_at
else:
raise NotImplementedError(
"after={!r} not supported.".format(after),
)
return partial(enqueue, after, function)
queue = django_rq.get_queue(
context['queue'],
)
return partial(_enqueue, queue, function)
def handle(self, queue='default', *args, **options):
if "." in queue:
connection_name, queue = queue.split('.')
else:
connection_name = 'default'
scheduler = get_scheduler(connection_name, queue, options.get('interval'))
scheduler.run()
def schedule_once(interval, timeout=None):
"""
Schedule job once or reschedule when interval changes
"""
rq_scheduler = django_rq.get_scheduler('default')
jobs = rq_scheduler.get_jobs()
functions = defaultdict(lambda: list())
map(lambda x: functions[x.func].extend([x.meta.get('interval'), x.timeout,]), jobs)
if not timeout:
default_timeout = global_settings.RQ_QUEUES.get('DEFAULT_TIMEOUT')
if default_timeout:
timeout = default_timeout
else:
timeout = 360
if f not in functions or interval not in functions[f] or functions[f][1] != timeout or len(functions[f]) > 2:
logger.info('Rescheduling job {} with interval: {}s'.format(f.func_name, interval))
# clear all scheduled jobs for this function
map(rq_scheduler.cancel, filter(lambda x: x.func==f, jobs))
# schedule with new interval and timeout
return rq_scheduler.schedule(datetime.datetime.now(), f, interval=interval, timeout=timeout)
else:
logger.info('Job already scheduled every {}s: {}'.format(interval, f.func_name))
def actions(request):
scheduler = get_scheduler()
if request.method == 'POST' and request.POST.get('action', False):
# Confirmation page for selected Action
if request.POST.get('_selected_action', False):
context_data = {
'action': request.POST['action'],
'job_ids': request.POST.getlist('_selected_action'),
}
return render(request, 'rq_scheduler/templates/confirm_action.html', context_data)
# Performing the actual action
elif request.POST.get('job_ids', False):
job_ids = request.POST.getlist('job_ids')
if request.POST['action'] == 'delete':
for job_id in job_ids:
job = Job.fetch(job_id, connection=scheduler.connection)
job.cancel()
messages.info(request, 'You have successfully deleted %s jobs!' % len(job_ids))
elif request.POST['action'] == 'enqueue':
for job_id in job_ids:
job = Job.fetch(job_id, connection=scheduler.connection)
scheduler.enqueue_job(job)
messages.info(request, 'You have successfully enqueued %d jobs!' % len(job_ids))
return redirect('rq_scheduler:jobs')
def jobs(request):
scheduler = get_scheduler()
jobs_list = scheduler.get_jobs(with_times=True)
items_per_page = 100
num_jobs = len(jobs_list)
page = int(request.GET.get('page', 1))
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
jobs_list = jobs_list[offset:offset + items_per_page]
for job, scheduled_at in jobs_list:
job.scheduled_at = make_aware(scheduled_at, utc)
job.created_at = make_aware(job.created_at, utc)
jobs = [job for job, scheduled_time in jobs_list]
else:
jobs = []
page_range = []
context_data = {
'title': 'Scheduled Jobs',
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
}
return render(request, 'rq_scheduler/templates/jobs.html', context_data)
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
scheduler = get_scheduler(
name=options.get('queue'), interval=options.get('interval'))
scheduler.run()
def enqueue_at(target_date, scheduler_name='default', *args, **kwargs):
function_name = '{0}.{1}'.format(f.__module__, f.__name__)
encrypted_args = [secure_serializer.dumps(function_name)]
encrypted_args += [secure_serializer.dumps(args)]
encrypted_args += [secure_serializer.dumps(kwargs)]
scheduler = django_rq.get_scheduler(scheduler_name)
proxy_method_name = '{}.{}'.format(secure_job_proxy.__module__, secure_job_proxy.__name__)
return scheduler.enqueue_at(target_date, proxy_method_name, *encrypted_args, **{})
def ready(self):
"""Do this when the app is ready."""
scheduler = django_rq.get_scheduler('default')
scheduler.schedule(
scheduled_time=datetime.utcnow(),
func='dashboard.jobs.generate_dashboard',
interval=10 * 60,
result_ttl=0
)
return True
def Expire_Expired():
if not thirtythirty.hdd.drives_are_unlocked():
logger.warning('Drives not unlocked yet.')
Sched = django_rq.get_scheduler('default')
Sched.enqueue_in(datetime.timedelta(hours=1), Expire_Expired)
exit()
for E in addressbook.address.Address.objects.filter(expires__lt=datetime.date.today()):
logger.debug('Contact %s has an expired key' % E)
E.user_state = addressbook.address.Address.FAIL
E.save()
def remove_automatic_scheduled_jobs(sch_id):
# cancelando o job automatico
scheduler = django_rq.get_scheduler('default')
list_of_job_instances = scheduler.get_jobs()
result = False
for job in list_of_job_instances:
if str(job.func_name) =='flyerapp.views.auto_schedule_search' and job.kwargs['id'] == sch_id:
scheduler.cancel(job)
result = True
return result
def check_status(self):
try:
scheduler = get_scheduler('default')
if not scheduler:
self.add_error(ServiceUnavailable("No scheduler"))
jobs = scheduler.get_jobs()
if len(jobs) == 0:
self.add_error(ServiceUnavailable("No jobs in scheduler"))
except BaseException as e:
self.add_error(ServiceUnavailable("Unknown error"), e)
raise(e)
def test_get_scheduler(self):
"""
Ensure get_scheduler creates a scheduler instance with the right
connection params for `test` queue.
"""
config = QUEUES["test"]
scheduler = get_scheduler("test")
connection_kwargs = scheduler.connection.connection_pool.connection_kwargs
self.assertEqual(scheduler.queue_name, "test")
self.assertEqual(connection_kwargs["host"], config["HOST"])
self.assertEqual(connection_kwargs["port"], config["PORT"])
self.assertEqual(connection_kwargs["db"], config["DB"])
def clear_jobs(request):
scheduler = get_scheduler()
if request.method == 'POST':
for job in scheduler.get_jobs():
job.cancel()
messages.info(request, 'You have successfully cleared all scheduled jobs')
return redirect('rq_scheduler:jobs')
context_data = {
'title': 'Are you sure?',
}
return render(request, 'rq_scheduler/templates/clear_jobs.html', context_data)
def register_job_schedule():
scheduler = django_rq.get_scheduler()
for job_id, params in job_schedule.items():
scheduler.cron(
params['cron_string'],
id=job_id,
func=import_string(params['func']),
timeout=params.get('timeout', DEFAULT_JOB_TIMEOUT)
)
for job in scheduler.get_jobs():
if job.id not in job_schedule:
scheduler.cancel(job)
def test_data_capture_scheduler_app_adds_cron_job(self):
config = apps.get_app_config('data_capture')
self.assertIsInstance(config, DataCaptureSchedulerApp)
config.ready() # call the ready() method
scheduler = django_rq.get_scheduler('default')
jobs = scheduler.get_jobs()
job_list = list(jobs)
self.assertEqual(len(job_list), 1)
the_job = job_list[0]
self.assertEqual(
the_job.func,
periodic_jobs.send_admin_approval_reminder_email)
self.assertEqual(the_job.meta['cron_string'], "* 12 * * MON")
def ready(self):
# Only register tasks if RQ Scheduler process
if "rqscheduler" not in sys.argv:
return
scheduler = django_rq.get_scheduler("default")
# Delete any existing jobs in the scheduler when the app starts up
for job in scheduler.get_jobs():
job.delete()
# Queue tasks
streams_tasks(scheduler)
def automatic_exec(request, *args, **kwargs):
sch_id = request.POST.get('id',None)
schedule = Schedule.objects.filter(pk=sch_id, active=1, logic_delete=False, departure_date__gte=datetime.now()).get()
if schedule:
scheduler = django_rq.get_scheduler('default')
scheduler.schedule(
scheduled_time=datetime.utcnow(),
func=auto_schedule_search,
# args=[],
kwargs={'id':sch_id},
interval=30,
repeat=None
)
def save(self, *args, **kwargs):
r = super(Content, self).save(*args, **kwargs)
if self.dirty:
if self.date <= datetime.now():
self.blog.dirty = True
self.blog.save()
else:
scheduler = django_rq.get_scheduler('default')
scheduler.enqueue_at(
self.date,
blog_sync.delay,
kwargs={'blog_id': self.blog.id}
)
return r
def enqueue_job(request, job_id):
scheduler = get_scheduler()
job = Job.fetch(job_id, connection=scheduler.connection)
if request.method == 'POST':
scheduler.enqueue_job(job)
messages.info(request, 'You have successfully enqueued %s' % job.id)
return redirect('rq_scheduler:jobs')
context_data = {
'title': 'Are you sure?',
'job': job,
}
return render(request, 'rq_scheduler/templates/enqueue_job.html', context_data)
def handle(self, *args, **settings):
Raw = sys.stdin.read()
Msg = email.message_from_string(Raw)
MType = addressbook.utils.msg_type(Raw)
Payload = emailclient.utils.text_payload(Msg)
Queue = django_rq.get_queue()
if not 'X-Lookingglass-Overhead' in Msg:
logger.error("Message doesn't have our X-header, is this actually destined for the user?")
exit(-1)
if Msg['Message-Id'] in Queue.job_ids:
logger.error('Already inserted this message into queue.')
exit(-1)
if MType == 'PGP-CLEARSIGN':
From_Addr = addressbook.address.objects.message_detective(Msg)
if not From_Addr:
logger.error('Unknown GPG clearsign')
exit(-1)
# bcast message -> Queue
elif MType == 'AXO-HS':
From_Addr = addressbook.address.objects.message_detective(Msg)
if not From_Addr:
logger.info('Got an unanticipated handshake from %s' % Msg['Reply-To'])
KSP = django_rq.enqueue(queue.keyserver.Pull, email=Msg['Reply-To'])
django_rq.enqueue(queue.address.Got_Handshake, fingerprint=KSP.result, handshake=Payload,
job_id=Msg['Message-Id'],
depends_on=RST,
)
exit()
if 'X-Lookingglass-Address-Reset' in Msg:
RST = django_rq.enqueue(queue.address.Reset, fingerprint=From_Addr.fingerprint)
django_rq.enqueue(queue.address.Got_Handshake, fingerprint=From_Addr.fingerprint, handshake=Payload,
job_id=Msg['Message-Id'],
depends_on=RST,
)
exit()
elif 'X-Lookingglass-Axo-Loop' in Msg:
Sched = django_rq.get_scheduler('default')
Sched.enqueue_in(datetime.timedelta(minutes=30),
queue.address.Got_Handshake, fingerprint=From_Addr.fingerprint, handshake=Payload,
job_id=Msg['Message-Id'])
exit()
django_rq.enqueue(queue.address.Got_Handshake, fingerprint=From_Addr.fingerprint, handshake=Payload,
job_id=Msg['Message-Id'])
def showStats(self):
import pprint
q = django_rq.get_queue()
qp = django_rq.get_queue('needs_passphrase')
s = django_rq.get_scheduler()
pp = pprint.PrettyPrinter()
print
print 'Job Queue:'
pp.pprint( q.jobs )
pp.pprint( qp.jobs )
print
print 'Scheduled tasks:'
pp.pprint( s.get_jobs(with_times=True) )
print
def setUp(self):
""" Create a superuser and log in """
self.user = AdminUserFactory(email='*****@*****.**')
self.client.login(email='*****@*****.**', password='******')
self.subject = "[Test] RQSchedulerViewExtensionsTestCase"
self.message = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean lobortis ornare vestibulum. Sed euismod euismod mattis. Suspendisse potenti. Vestibulum eget faucibus lacus. Quisque in eros augue. Sed diam lorem, finibus congue auctor vel, volutpat a lacus. Proin ut pellentesque nisi, ut dignissim erat. Donec fringilla venenatis est, a tempor turpis tempus a. Praesent eu magna lectus."
self.from_email = "Testbot <test@%s>" % settings.PROJECT_DOMAIN
self.recipient_list = ["*****@*****.**"]
self.scheduler = get_scheduler()
self.western = timezone('America/Los_Angeles')
self.scheduled_time = self.western.localize(datetime(2020, 1, 1))
self.scheduler.enqueue_at(self.scheduled_time, send_mail, self.subject, self.message, self.from_email, self.recipient_list)
def handle(self, *args, **options):
interval = options.get('interval')
scheduler = get_scheduler(
name=options.get('queue'), interval=interval)
while 1:
try:
scheduler.run()
break
except ValueError as err:
if options.get('retry', False) and 'already an active RQ scheduler' in str(err):
msg = 'RQ scheduler already exists, retrying in {} seconds'.format(interval)
scheduler.log.warning(msg)
time.sleep(interval)
else:
raise
def ready(self):
import django_rq
from SchedulerDemo.tasks import change_coupon_type
scheduler = django_rq.get_scheduler('default')
print(scheduler.get_jobs())
if bool(scheduler.get_jobs()):
pass
else:
job = scheduler.schedule(
scheduled_time=datetime.datetime.now(),
func=change_coupon_type,
interval=70,
)
print (scheduler.get_jobs())
def start_all_jobs():
"""
Inicia los checkeos cada "TIEMPO_ENTRE_CHECKS".
Note que en "check_uptime.check_uptime" hay un verificador que comprueba
que los checkeos no se den dentro de un intervalo menor a cierto número de
minutos.
"""
from .models import Site
sites = Site.objects.filter(active=True)
scheduler = django_rq.get_scheduler('default')
for site in sites:
scheduler.schedule(now(), check_uptime, args=(site.slug,), repeat=None, interval=TIEMPO_ENTRE_CHECKS)
def update(self, instance, validated_data):
super().update(instance, validated_data)
status = validated_data.get('status', None)
if status == 'COMPLETED':
new_order = Order.objects.create(
contacts=instance.contact,
sale_rep=self.context.get('request').user,
campaign=instance.campaign)
steps = new_order.campaign.follow_up_plan.steps.all()
step_details = [
StepDetail(step=s, order=new_order, information={})
for s in steps
]
step_details[len(step_details) - 1].information = {
"Choose Packages": {
"type": 'final',
"result": {}
}
}
step_details[len(step_details) -
1].information['Choose Packages']['result'] = {
f'{p.id}': {}
for p in instance.campaign.packages.all()
}
step_details = StepDetail.objects.bulk_create(step_details)
for step_detail in step_details:
job = django_rq.get_scheduler('default')
end_date = step_detail.created + \
timedelta(days=step_detail.step.duration)
timestamp1 = calendar.timegm((end_date).timetuple())
end_date = datetime.utcfromtimestamp(timestamp1)
job.enqueue_at(end_date, check_step_completed, step_detail)
if "Send Email" in steps[0].actions:
cur_contact = step_details[0].order.contacts
queue = django_rq.get_queue('default', is_async=True)
queue.enqueue(
send_email_api_step,
self.context.get('request').user, cur_contact.mail,
"*****@*****.**", steps[0].mail_template.subject,
handle_mail_template.manipulate_template(
steps[0].mail_template.template,
contact=cur_contact,
packages=step_details[0].order.campaign.packages.all(
)), step_details[0].id)
return instance
def setUp(self):
""" Create a superuser and log in """
self.user = AdminUserFactory(email='*****@*****.**')
self.client.login(email='*****@*****.**', password='******')
self.subject = "[Test] RQSchedulerViewExtensionsTestCase"
self.message = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean lobortis ornare vestibulum. Sed euismod euismod mattis. Suspendisse potenti. Vestibulum eget faucibus lacus. Quisque in eros augue. Sed diam lorem, finibus congue auctor vel, volutpat a lacus. Proin ut pellentesque nisi, ut dignissim erat. Donec fringilla venenatis est, a tempor turpis tempus a. Praesent eu magna lectus."
self.from_email = "Testbot <test@%s>" % settings.PROJECT_DOMAIN
self.recipient_list = ["*****@*****.**"]
self.scheduler = get_scheduler()
self.western = timezone('America/Los_Angeles')
self.scheduled_time = self.western.localize(datetime(2020, 1, 1))
self.scheduler.enqueue_at(self.scheduled_time, send_mail, self.subject,
self.message, self.from_email,
self.recipient_list)
def get_pending_tasks(self, regex=None):
import django_rq
queue = django_rq.get_queue(self.DEFAULT_QUEUE_NAME)
scheduler = django_rq.get_scheduler(self.DEFAULT_QUEUE_NAME)
if regex is None:
return queue.get_job_ids() + [x.id for x in scheduler.get_jobs()]
else:
import re
_ids = []
_jobs = queue.get_jobs() + scheduler.get_jobs()
for _job in _jobs:
_s = re.search(regex, _job.func_name)
if _s:
_ids.append(_job.id)
return _ids
def ready(self):
from actstream import registry
from .signals import incentive_for_job_approved
registry.register(self.get_model('Job'))
import django_rq
from messaging.scheduled_tasks import broadcast_approved_jobs
scheduler = django_rq.get_scheduler("default")
# Delete any existing jobs in the scheduler when the app starts up
for job in scheduler.get_jobs():
job.delete()
scheduler.cron(
cron_string="30 4 * * *",
func=broadcast_approved_jobs, # Function to be queued
queue_name=scheduler.queue_name)
def form_valid(self, form):
form_data = form.cleaned_data
message = form_data['message']
email = form_data['email']
scheduler = django_rq.get_scheduler('default')
scheduler.enqueue_at(datetime.utcnow(),
func=send_message_job,
message=message,
email=email)
scheduler.enqueue_at(datetime.utcnow(),
func=send_push_admin_job,
email=email)
scheduler.enqueue_at(datetime.utcnow(),
func=send_push_user_job,
email=[email])
return super().form_valid(form)
def handle(self, *args, **options):
scheduler = django_rq.get_scheduler('default')
self.stdout.write('Clearing previous scheduled jobs...')
for job in scheduler.get_jobs():
job.cancel()
self.stdout.write(u'Scheduling "check_all_hosts" task...')
scheduler.schedule(
scheduled_time=datetime.utcnow(),
func=check_all_hosts,
interval=60,
repeat=None,
)
self.stdout.write('Done.')
def automatic_exec(request, *args, **kwargs):
sch_id = request.POST.get('id', None)
schedule = Schedule.objects.filter(
pk=sch_id,
active=1,
logic_delete=False,
departure_date__gte=datetime.now()).get()
if schedule:
scheduler = django_rq.get_scheduler('default')
scheduler.schedule(
scheduled_time=datetime.utcnow(),
func=auto_schedule_search,
# args=[],
kwargs={'id': sch_id},
interval=30,
repeat=None)
def export_project(task_id, user, dst_format=None, server_url=None):
try:
db_task = Task.objects.get(pk=task_id)
if not dst_format:
dst_format = DEFAULT_FORMAT
cache_dir = get_export_cache_dir(db_task)
save_dir = osp.join(cache_dir, dst_format)
archive_path = osp.normpath(save_dir) + '.zip'
task_time = timezone.localtime(db_task.updated_date).timestamp()
if not (osp.exists(archive_path) and \
task_time <= osp.getmtime(archive_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir,
prefix=dst_format +
'_') as temp_dir:
project = TaskProject.from_task(db_task, user)
project.export(dst_format,
save_dir=temp_dir,
save_images=True,
server_url=server_url)
os.makedirs(cache_dir, exist_ok=True)
make_zip_archive(temp_dir, archive_path)
archive_ctime = osp.getctime(archive_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL,
func=clear_export_cache,
task_id=task_id,
file_path=archive_path,
file_ctime=archive_ctime)
slogger.task[task_id].info(
"The task '{}' is exported as '{}' "
"and available for downloading for next '{}'. "
"Export cache cleaning job is enqueued, "
"id '{}', start in '{}'".format(db_task.name, dst_format,
CACHE_TTL, cleaning_job.id,
CACHE_TTL))
return archive_path
except Exception:
log_exception(slogger.task[task_id])
raise
def check_status(job: Union[str, Job], scheduler=None):
_scheduler = scheduler
if not scheduler:
_scheduler = django_rq.get_scheduler("default")
if not isinstance(job, Job):
try:
job = Job.fetch(job, connection=_scheduler.connection)
except NoSuchJobError:
return None
scheduled = scheduled_execution_time(job.get_id(), scheduler)
if scheduled:
return job.get_status() or "scheduled"
else:
return job.get_status() or "cancelled"
def CampaignSchedule(request, pk):
campaign = get_object_or_404(Campaign, pk=pk)
scheduler = django_rq.get_scheduler('default')
job = scheduler.schedule(
scheduled_time=datetime.datetime.now(),
func=main.extras.TwitterBot.twitterBot,
args=[campaign.pk],
interval=3600,
timeout=3600,
repeat=None
)
campaign.jobId = job.id
campaign.save()
return redirect(campaign)
def schedule_jobs(self):
try:
import django_rq
scheduler = django_rq.get_scheduler('cron')
# Cron task to check status of companies
scheduler.cron(
"*/5 * * * *", # Run every 5 minutes
func=notify_about_unread_messages,
timeout=settings.RQ_QUEUES['cron']['DEFAULT_TIMEOUT'])
except ImportError:
# django RQ couldn't be imported, the email notifications won't work
pass
except KeyError:
# 'cron' not found in RQ queues
# TODO: create whistle setting for this
pass
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
scheduler = get_scheduler(name=options.get('queue'), interval=options.get('interval'))
cron_jobs = getattr(settings, 'RQ_CRON_JOBS')
scheduled_job_ids = []
for cron_job in cron_jobs:
func = cron_job['func']
if isinstance(func, str):
module, sep, attribute = func.rpartition('.')
mod = import_module(module)
cron_job['func'] = getattr(mod, attribute)
job_id = f'{cron_job['func'].__module__}.{cron_job['func'].__name__}'
scheduled_job_ids.append(job_id)
if job_id in scheduler:
# dequeue job if it had been initially queued
job = scheduler.job_class.fetch(job_id, connection=scheduler.connection)
scheduler.cancel(job)
logger.info(f'Scheduling job {job_id} with schedule {cron_job["cron_string"]}')
scheduler.cron(
cron_string=cron_job.get('cron_string'),
func=cron_job.get('func'),
args=cron_job.get('args'),
kwargs=cron_job.get('kwargs'),
repeat=cron_job.get('repeat'),
queue_name=cron_job.get('queue_name', 'default'),
id=job_id,
)
# cancel all jobs that are not in the settings file currently
bad_cron_jobs = [
job for job in scheduler.get_jobs()
if job.id not in scheduled_job_ids
]
for job in bad_cron_jobs:
scheduler.cancel(job)
scheduler.run()
def ready(self):
"""
On django runtime, load up the following code.
"""
scheduler = django_rq.get_scheduler('default')
# Delete previously loaded ETLs.
for job in scheduler.get_jobs():
if "shared_etl" in str(
job): # Only delete jobs belonging to this app.
job.delete()
# Variable used to track the maximum number of minutes the ETL can
# run before it's considered an error and needs to stop the ETL.
timeout = timedelta(minutes=999)
# DEPRECATED
# scheduler.cron(
# "0 0 1 * *", # A cron string - Run every 12:00AM on the FIRST day of every month
# func=run_update_ongoing_orders_func, # Function to be queued
# repeat=None, # Repeat this number of times (None means repeat forever)
# timeout=timeout.seconds # Automatically terminate process if exceeds this time.
# )
# scheduler.cron(
# "0 0 L * *", # A cron string - Run every 12:00AM on the LAST day of every month
# func=run_update_ongoing_orders_func, # Function to be queued
# repeat=None, # Repeat this number of times (None means repeat forever)
# timeout=timeout.seconds # Automatically terminate process if exceeds this time.
# )
scheduler.cron(
"1 0 * * *", # A cron string - Run every 12:00AM every day.
func=run_update_away_logs_func, # Function to be queued
repeat=
None, # Repeat this number of times (None means repeat forever)
timeout=timeout.
seconds # Automatically terminate process if exceeds this time.
)
scheduler.cron(
"1 0 * * *", # A cron string - Run every 12:00AM every day.
func=run_update_expired_associates_func, # Function to be queued
repeat=
None, # Repeat this number of times (None means repeat forever)
timeout=timeout.
seconds # Automatically terminate process if exceeds this time.
)
def ready(self):
# Only register tasks if RQ Scheduler process
if "rqscheduler" not in sys.argv:
return
from thefederation.social import make_daily_post
from thefederation.tasks import aggregate_daily_stats
from thefederation.tasks import clean_duplicate_nodes
from thefederation.tasks import fill_country_information
from thefederation.tasks import poll_nodes
scheduler = django_rq.get_scheduler()
# Delete any existing jobs in the scheduler when the app starts up
for job in scheduler.get_jobs():
job.delete()
scheduler.schedule(
scheduled_time=datetime.datetime.utcnow(),
func=aggregate_daily_stats,
interval=5500,
queue_name='high',
)
scheduler.cron(
'0 10 * * *',
func=make_daily_post,
queue_name='high',
)
scheduler.cron(
'18 4 * * *',
func=clean_duplicate_nodes,
queue_name='medium',
timeout=3600,
)
scheduler.cron(
'23 6 * * *',
func=fill_country_information,
queue_name='low',
timeout=3600,
)
scheduler.schedule(
scheduled_time=datetime.datetime.utcnow(),
func=poll_nodes,
interval=10800,
queue_name='medium',
)
def list(self, request, *args, **kwargs):
import django_rq
scheduler = django_rq.get_scheduler('default')
jobs_and_times = scheduler.get_jobs(with_times=True)
filters = Q()
filters.add(Q(user=request.user.id), Q.AND)
filters.add(Q(is_public=True), Q.OR)
name = request.query_params.get('name', None)
if name:
filters.add(Q(name__icontains=name), Q.AND)
query = self.get_queryset().filter(filters)
serializer = self.get_serializer(query, many=True)
new_data = {"data": serializer.data, "total": len(serializer.data)}
return Response(new_data, status=status.HTTP_200_OK)
def create(self, validated_data):
license = super().create(validated_data)
scheduler = django_rq.get_scheduler('default')
user = User.objects.get(username=self.context.get('request').user)
remind_date = license.start_date + \
timedelta(days=license.duration*30) - timedelta(days=10)
timestamp1 = calendar.timegm((remind_date).timetuple())
start_date = datetime.utcfromtimestamp(timestamp1)
product = None
for f in license.package.features.all():
product = model_to_dict(f.product)
break
scheduler.enqueue_at(
start_date, send_email, user,
f'License Reminder (Order #{license.order.id})',
f'Product: {product["name"]}\r\nPackage: {license.package.name} ({license.duration} month(s))\r\nYour license will be expired in 10 days'
)
return license
def form_valid(self, form):
url = form.cleaned_data['url']
schedule_times = form.cleaned_data.get('schedule_times')
schedule_interval = form.cleaned_data.get('schedule_interval')
if schedule_times and schedule_interval:
# Schedule the job with the form parameters
scheduler = django_rq.get_scheduler('default')
job = scheduler.schedule(
scheduled_time=datetime.datetime.now(),
func=scheduled_get_url_words,
args=[url],
interval=schedule_interval,
repeat=schedule_times,
)
else:
# Just execute the job asynchronously
get_url_words.delay(url)
return super(TasksHomeFormView, self).form_valid(form)
def scheduled_execution_time(job_id, scheduler=None, naive=True):
"""Get RQ-Scheduler scheduled execution time for specific job."""
_scheduler = scheduler
if not scheduler:
_scheduler = django_rq.get_scheduler("default")
# Scheduler keeps jobs in a single key, they are sorted by score, which is
# scheduled execution time (linux epoch). We can retrieve single
# entry's score.
time = _scheduler.connection.zscore(_scheduler.scheduled_jobs_key, job_id)
# Convert linux time epoch to UTC.
if time:
time = from_unix(time)
if not naive:
# By default, RQ-Scheduler uses UTC naive (TZ-unaware) objects,
# which we can "convert" to TZ-aware UTC.
time = time.replace(tzinfo=pytz.UTC)
return time
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
scheduler = get_scheduler(name=options.get('queue'),
interval=options.get('interval'))
self.stdout.write("Waiting to acquire lock...")
while True:
try:
scheduler.run()
break
except ValueError, exc:
if exc.message == "There's already an active RQ scheduler":
time.sleep(SCHEDULER_INTERVAL_SECONDS)
else:
raise
def export_task(task_id, dst_format, server_url=None, save_images=False):
try:
db_task = Task.objects.get(pk=task_id)
cache_dir = get_export_cache_dir(db_task)
exporter = EXPORT_FORMATS[dst_format]
output_base = '%s_%s' % ('dataset' if save_images else 'annotations',
make_file_name(to_snake_case(dst_format)))
output_path = '%s.%s' % (output_base, exporter.EXT)
output_path = osp.join(cache_dir, output_path)
task_time = timezone.localtime(db_task.updated_date).timestamp()
if not (osp.exists(output_path) and \
task_time <= osp.getmtime(output_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir:
temp_file = osp.join(temp_dir, 'result')
task.export_task(task_id,
temp_file,
dst_format,
server_url=server_url,
save_images=save_images)
os.replace(temp_file, output_path)
archive_ctime = osp.getctime(output_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL,
func=clear_export_cache,
task_id=task_id,
file_path=output_path,
file_ctime=archive_ctime)
slogger.task[task_id].info(
"The task '{}' is exported as '{}' at '{}' "
"and available for downloading for the next {}. "
"Export cache cleaning job is enqueued, id '{}'".format(
db_task.name, dst_format, output_path, CACHE_TTL,
cleaning_job.id))
return output_path
except Exception:
log_exception(slogger.task[task_id])
raise
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
# Verbosity is defined by default in BaseCommand for all commands
verbosity = options.get('verbosity')
if verbosity >= 2:
level = 'DEBUG'
elif verbosity == 0:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = get_scheduler(name=options.get('queue'),
interval=options.get('interval'))
scheduler.run()
def ready(self):
scheduler = django_rq.get_scheduler('default')
# Delete any existing jobs in the scheduler when the app starts up
for job in scheduler.get_jobs():
job.delete()
# Have 'db_relocation' run every 15 minutes
print(
"db_relocator was registered in the scheduler with 15 min interval"
)
scheduler.schedule(datetime.datetime.now(datetime.timezone.utc),
db_relocator,
interval=60 * 15)
# Have 'r_db_clean' run every 30 days
print(
"r_db_clean was registered in the scheduler with 30 days interval")
scheduler.schedule(datetime.datetime.now(datetime.timezone.utc),
r_db_clean,
interval=60 * 60 * 24 * 30)
def post(self, request, *args, **kwargs):
form = SignUpForm(request.data)
if form.is_valid():
form.save()
username = form.cleaned_data.get('username')
raw_password = form.cleaned_data.get('password1')
mail = form.cleaned_data.get('email')
user = authenticate(username=username, password=raw_password)
otus_user = OtusUser.objects.create(user=user)
login(request, user)
scheduler = get_scheduler('default')
scheduler.schedule(datetime.now(),
periodic_send,
interval=60,
args=[otus_user.id, mail])
smail.delay(mail)
return Response({'success': True})
else:
return Response({'success': False, 'errors': form.errors})
def delete_org(scratch_org):
"""Delete a scratch org by deleting its ActiveScratchOrg record
in the Dev Hub org."""
devhub_username = scratch_org.owner_sf_username
org_id = scratch_org.config["org_id"]
devhub_api = get_devhub_api(devhub_username=devhub_username)
records = (
devhub_api.query(
f"SELECT Id FROM ActiveScratchOrg WHERE ScratchOrg='{org_id}'").
get("records")
# the above could return an empty list, so we have to use an or:
or [{}])[0]
active_scratch_org_id = records.get("Id")
if active_scratch_org_id:
devhub_api.ActiveScratchOrg.delete(active_scratch_org_id)
if scratch_org.expiry_job_id:
scheduler = get_scheduler("default")
scheduler.cancel(scratch_org.expiry_job_id)
def try_to_end_match(id):
match = models.Match.objects.get(id=id)
region = match.region
# Fetch the match information
try:
match_info = api.fetch_match(match_api_id=match.api_id, region=region)
except exceptions.HTTPError:
logger.error('HTTP error', exc_info=True)
return
# Get the duration in seconds
duration = match_info.get('gameDuration')
# Can't do anything if the game hasn't ended yet
if duration is None:
logger.info(f'Match {match.id} has not finished')
return
# Set the match's end time
match.duration = int(duration)
# Set the winning team ID
if match_info['teams'][0]['win'] == 'Win':
match.winning_team_id = match_info['teams'][0]['teamId']
else:
match.winning_team_id = match_info['teams'][1]['teamId']
ok = False
while not ok:
ok = models.CremeModel.objects.get(name=settings.MODEL_NAME)\
.fit_one(match.raw_info, match.duration)
if not ok:
logger.warning('Optimistic logging failed when fitting')
# Stop polling the match
scheduler = django_rq.get_scheduler('default')
scheduler.cancel(match.rq_job_id)
match.rq_job_id = None
match.save()
def handle(self, *args, **settings):
if settings['test']:
print 'here we go!'
NP = django_rq.get_queue('needs_passphrase')
DF = django_rq.get_queue()
NP.enqueue(test_task)
DF.enqueue(test_secret_task)
print 'did it!'
elif settings['register']:
queue.keyserver.Push.delay()
elif settings['pull']:
django_rq.enqueue(queue.keyserver.Pull, covername=settings['pull'])
elif settings['flush_schedule']:
S = django_rq.get_scheduler()
for Task in S.get_jobs():
S.cancel(Task)
self.showStats()
elif settings['unlock']:
queue.hdd.Unlock.delay()
else:
self.showStats()
def job_detail(request, job_id):
scheduler = get_scheduler()
try:
job = Job.fetch(job_id, connection=scheduler.connection)
except NoSuchJobError:
raise Http404("Couldn't find job with this ID: %s" % job_id)
if job in scheduler:
jobs_list_with_times = scheduler.get_jobs(with_times=True)
jobs_list = [j for j, scheduled_at in jobs_list_with_times]
job.scheduled_at = jobs_list_with_times[jobs_list.index(job)][1]
job.scheduled_at = make_aware(job.scheduled_at, utc)
job.created_at = make_aware(job.created_at, utc)
job.enqueued_at = make_aware(job.enqueued_at, utc) if job.enqueued_at else None
context_data = {
'title': 'Job Info',
'job': job,
}
return render(request, 'rq_scheduler/templates/job_detail.html', context_data)
def test_callable_args_and_kwagrgs(self):
job = self.JobClassFactory(
callable='scheduler.tests.test_args_kwargs')
date = timezone.now()
JobArgFactory(arg_name='str_val',
str_val='one',
content_object=job)
JobKwargFactory(key='key1',
arg_name='int_val',
int_val=2,
content_object=job)
JobKwargFactory(key='key2',
arg_name='datetime_val',
datetime_val=date,
content_object=job)
job.save()
scheduler = django_rq.get_scheduler(job.queue)
entry = next(i for i in scheduler.get_jobs() if i.id == job.job_id)
self.assertEqual(
entry.perform(),
"test_args_kwargs('one', key1=2, key2={})".format(repr(date)))
