def handle(self, *args, **kwargs):
print(json.dumps(json.loads(autoscale(None).content), indent=2))
for recipe in Recipe.objects.filter(
active=True,
job_utm__lt=utc_milliseconds()).exclude(job_utm=0):
print(recipe.id, recipe.name, recipe.get_days())
print('---')
for recipe in Recipe.objects.filter(
active=True,
worker_utm__gte=utc_milliseconds() - JOB_LOOKBACK_MS):
print(recipe.id, recipe.name, recipe.worker_uid)
def setUp(self):
self.account = account_create()
self.project = project_create()
now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles')
today_tz = now_tz.strftime('%a')
yesterday_tz = (now_tz - timedelta(hours=24)).strftime('%a')
tomorrow_tz = (now_tz + timedelta(hours=24)).strftime('%a')
self.recipe_today = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_TODAY',
active=True,
week=json.dumps([today_tz]),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([
{
"tag": "hello",
"values": {},
"sequence": 1
},
]),
worker_uid="OTHER_WORKER",
worker_utm=utc_milliseconds() - (
JOB_LOOKBACK_MS * 10
) # important for test ( jobs older than this get job_done flag reset )
)
self.recipe_not_today = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_NOT_TODAY',
active=True,
week=json.dumps([yesterday_tz, tomorrow_tz]),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([
{
"tag": "hello",
"values": {},
"sequence": 1
},
]),
worker_uid="OTHER_WORKER",
worker_utm=utc_milliseconds() - (
JOB_LOOKBACK_MS * 10
) # important for test ( jobs older than this get job_done flag reset )
)
def setUp(self):
self.account = account_create()
self.project = project_create()
self.recipe = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_MANUAL',
active = True,
manual = True,
week = [],
hour = [],
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "manual",
"values": {},
"sequence": 1
},
]),
)
now_tz = utc_to_timezone(datetime.utcnow(), self.recipe.timezone)
self.recipe_done = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_MANUAL',
active = True,
manual = True,
week = [],
hour = [],
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "manual",
"values": {},
"sequence": 1
},
]),
job_done = True,
job_status = json.dumps({
"day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"],
"date_tz": str(now_tz.date()),
"force": True,
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow()),
"script": "manual",
"hour": 0,
"stdout":"",
"stderr": "",
"done": True
}
]
}),
worker_uid = 'TEST_WORKER',
worker_utm = utc_milliseconds() - (JOB_LOOKBACK_MS * 10) # important for test ( jobs older than this get job_done flag reset )
)
def autoscale(request):
scale = {
'jobs': 0,
'workers': {
'jobs': settings.WORKER_JOBS,
'max': settings.WORKER_MAX,
'existing': 0,
'required': 0
}
}
# get task and worker list
scale['jobs'] = Recipe.objects.filter(
active=True,
job_utm__lt=utc_milliseconds()).exclude(job_utm=0).count()
scale['workers']['existing'] = 3 if request == 'TEST' else sum(
1 for instance in group_instances_list(('PROVISIONING', 'STAGING',
'RUNNING')))
scale['workers']['required'] = min(
settings.WORKER_MAX,
math.ceil(scale['jobs'] / scale['workers']['jobs']))
if request != 'TEST' and scale['workers']['required'] > scale['workers'][
'existing']:
group_instances_resize(scale['workers']['required'])
# log the scaling operation
log_manager_scale(scale)
return JsonResponse(scale)
def setUp(self):
self.account = account_create()
self.project = project_create()
self.recipe = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_MANUAL',
active = True,
manual = True,
week = [],
hour = [],
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "manual",
"values": {},
"sequence": 1
},
]),
)
self.recipe.update()
now_tz = utc_to_timezone(datetime.utcnow(), self.recipe.timezone)
self.recipe_done = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_MANUAL',
active = True,
manual = True,
week = [],
hour = [],
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "manual",
"values": {},
"sequence": 1
},
]),
job_status = json.dumps({
"date_tz": str(now_tz.date()),
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow()),
"script": "manual",
"hour": 0,
"stdout":"",
"stderr": "",
"done": True
}
]
}),
worker_uid = 'TEST_WORKER',
worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE
)
self.recipe_done.update()
def assertRecipeNotDone(cls, recipe): recipe.refresh_from_db() status = recipe.get_status() job_time = datetime.utcfromtimestamp(int(recipe.job_utm/1000)) cls.assertFalse(all(task['done'] == True for task in status['tasks'])) cls.assertLessEqual(recipe.job_utm, utc_milliseconds()) cls.assertEqual(job_time.minute, 0)
def setUp(self):
self.account = account_create()
self.project = project_create()
now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles')
self.job_done = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_DONE',
active = True,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
},
]),
job_status = json.dumps({
"day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"],
"date_tz": str(now_tz.date()),
"force": True,
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": now_tz.hour,
"stdout":"",
"stderr": "",
"done": True
}, {
"instance": 2,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": now_tz.hour,
"stdout":"",
"stderr": "",
"done": True
}
]
}),
job_done = False,
worker_uid = "SAMPLE_WORKER",
worker_utm = utc_milliseconds() - (JOB_LOOKBACK_MS * 2)
)
self.RECIPE_DONE = self.job_done.uid()
def test_time(self):
utc_now = datetime.utcnow().replace(second=0, microsecond=0)
tz_now = datetime.now(tz=pytz.timezone(self.recipe.timezone)).replace(second=0, microsecond=0)
utm = utc_milliseconds(utc_now)
tz1 = utc_milliseconds_to_timezone(utm, self.recipe.timezone)
tz2 = utc_to_timezone(utc_now, self.recipe.timezone)
tz3 = timezone_to_utc(tz_now)
self.assertEqual(tz1, tz2)
self.assertEqual(tz3, utc_now)
def worker_pull(worker_uid, jobs=1):
'''Atomic reservation of worker in jobs.
Args:
- worker_uid ( string ) - identifies a unique worker, must be same for every call from same worker.
- jobs ( integer ) - number of jobs to pull
'''
jobs_all = []
jobs_new = []
worker_utm = utc_milliseconds()
worker_lookback = worker_utm - JOB_LOOKBACK_MS
worker_recheck = worker_utm - JOB_RECHECK_MS
if jobs:
with transaction.atomic():
# every half hour put jobs back in rotation so worker can trigger get_status logic, triggers status at 24 hour mark
Recipe.objects.filter(
active=True, manual=False,
worker_utm__lt=worker_recheck).select_for_update(
skip_locked=True).update(job_done=False)
# find recipes that are available but have not been pinged recently from all workers ( pulls from pool )
where = Recipe.objects.filter(
job_done=False,
active=True,
worker_utm__lt=worker_lookback,
).select_for_update(
skip_locked=True).order_by('worker_utm').values_list(
'id', flat=True)[:jobs]
# mark those recipes as belonging to this worker
Recipe.objects.filter(id__in=where).update(worker_uid=worker_uid,
worker_utm=worker_utm)
# find all recipes that belong to this worker and check if they have new tasks
for job in Recipe.objects.filter(job_done=False,
active=True,
worker_uid=worker_uid):
jobs_all.append(job.id)
task = job.get_task() # also checks if job is done
if job.worker_utm == worker_utm and task: # jobs with current timestamp are new ( odds of a ping matching this worker_utm? ), isolate evens and odds?
jobs_new.append(task)
return jobs_all, jobs_new
def worker_ping(worker_uid, recipe_uids):
# update recipes that belong to this worker
if recipe_uids:
Recipe.objects.filter(
worker_uid=worker_uid,
id__in=recipe_uids).update(worker_utm=utc_milliseconds())
def setUp(self):
self.account = account_create()
self.project = project_create()
self.job_new = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_NEW',
active=True,
week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([
{
"tag": "hello",
"values": {
"say_first": "Hi Once",
"say_second": "Hi Twice",
"sleep": 0
},
"sequence": 1
},
]),
)
self.RECIPE_NEW = self.job_new.uid()
self.job_expired = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_EXPIRED',
active=True,
week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([
{
"tag": "hello",
"values": {
"say_first": "Hi Once",
"say_second": "Hi Twice",
"sleep": 0
},
"sequence": 1
},
]),
job_done=False,
worker_uid="SAMPLE_WORKER",
worker_utm=utc_milliseconds() - (JOB_LOOKBACK_MS * 2))
self.RECIPE_EXPIRED = self.job_expired.uid()
self.job_running = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_RUNNING',
active=True,
week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([{
"tag": "hello",
"values": {
"say_first": "Hi Once",
"say_second": "Hi Twice",
"sleep": 0
},
"sequence": 1
}]),
job_done=False,
worker_uid="OTHER_WORKER",
worker_utm=utc_milliseconds())
self.RECIPE_RUNNING = self.job_running.uid()
self.job_paused = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_PAUSED',
active=False,
week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([{
"tag": "hello",
"values": {
"say_first": "Hi Once",
"say_second": "Hi Twice",
"sleep": 0
},
"sequence": 1
}]),
job_done=False,
worker_uid="OTHER_WORKER",
worker_utm=utc_milliseconds() - (JOB_LOOKBACK_MS * 10))
self.RECIPE_PAUSED = self.job_paused.uid()
# paused so its not part of the normal flow ( unpause to use in test )
self.job_error = Recipe.objects.create(
account=self.account,
project=self.project,
name='RECIPE_ERROR',
active=False,
week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour=json.dumps([0]),
timezone='America/Los_Angeles',
tasks=json.dumps([
{
"tag": "hello",
"values": {
"say_first": "Hi Once",
"say_second": "Hi Twice",
"sleep": 0,
"errror": "An error is triggered."
},
"sequence": 1
},
]),
job_done=False,
worker_uid="",
worker_utm=0)
self.RECIPE_ERROR = self.job_error.uid()
def test_job_utm(hour = 0, days_offset = 0): utc = datetime.utcnow().replace(hour=hour, minute=0, second=0, microsecond=0) if days_offset: utc += timedelta(days=days_offset) return utc_milliseconds(utc)
def setUp(self):
self.account = account_create()
self.project = project_create()
now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles')
status = {
"date_tz":str(now_tz.date()),
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_PENDING",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": False
}, {
"instance": 2,
"order": 2,
"event": "JOB_PENDING",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": False
},
]
}
self.job_new = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_NEW',
active = True,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
},
]),
)
self.RECIPE_NEW = self.job_new.uid()
self.job_queued = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_QUEUED',
active = True,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
},
]),
job_status = json.dumps(status.copy()),
job_utm = test_job_utm()
)
self.RECIPE_QUEUED = self.job_queued.uid()
self.job_expired = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_EXPIRED',
active = True,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
},
]),
job_status = json.dumps(status.copy()),
job_utm = test_job_utm(),
worker_uid = "SAMPLE_WORKER",
worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE
)
self.RECIPE_EXPIRED = self.job_expired.uid()
self.job_running = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_RUNNING',
active = True,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
}
]),
job_status = json.dumps(status.copy()),
job_utm = test_job_utm(),
worker_uid = "OTHER_WORKER",
worker_utm = utc_milliseconds()
)
self.RECIPE_RUNNING = self.job_running.uid()
self.job_paused = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_PAUSED',
active = False,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0},
"sequence": 1
}
]),
job_status = json.dumps(status.copy()),
job_utm = test_job_utm(),
worker_uid = "OTHER_WORKER",
worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE
)
self.RECIPE_PAUSED = self.job_paused.uid()
# paused so its not part of the normal flow ( unpause to use in test )
self.job_error = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_ERROR',
active = False,
week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0, "errror":"An error is triggered."},
"sequence": 1
},
]),
job_status = json.dumps(status.copy()),
job_utm = test_job_utm(),
worker_uid = "",
worker_utm = 0
)
self.RECIPE_ERROR = self.job_error.uid()
def setUp(self):
self.account = account_create()
self.project = project_create()
now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles')
today_tz = now_tz.strftime('%a')
yesterday_tz = (now_tz - timedelta(days=1)).strftime('%a')
tomorrow_tz = (now_tz + timedelta(days=1)).strftime('%a')
self.recipe_today = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_TODAY',
active = True,
week = json.dumps([today_tz]),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {},
"sequence": 1
},
]),
job_status = json.dumps({
"date_tz":str(now_tz.date()),
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_PENDING",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": False
},
{
"instance": 2,
"order": 0,
"event": "JOB_PENDING",
"utc":str(datetime.utcnow()),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": False
}
]
}),
worker_uid = "OTHER_WORKER",
worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE,
job_utm = test_job_utm()
)
self.recipe_not_today = Recipe.objects.create(
account = self.account,
project = self.project,
name = 'RECIPE_NOT_TODAY',
active = True,
week = json.dumps([yesterday_tz, tomorrow_tz]),
hour = json.dumps([0]),
timezone = 'America/Los_Angeles',
tasks = json.dumps([
{ "tag": "hello",
"values": {},
"sequence": 1
},
]),
job_status = json.dumps({
"date_tz":str(now_tz.date() - timedelta(days=1)),
"tasks": [
{
"instance": 1,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow() - timedelta(days=1)),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": True
},
{
"instance": 2,
"order": 0,
"event": "JOB_END",
"utc":str(datetime.utcnow() - timedelta(days=1)),
"script": "hello",
"hour": 0,
"stdout":"",
"stderr": "",
"done": True
}
]
}),
worker_uid = "OTHER_WORKER",
worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE,
job_utm = test_job_utm(0, 1)
)
