def handle(self, *args, **kwargs): print(json.dumps(json.loads(autoscale(None).content), indent=2)) for recipe in Recipe.objects.filter( active=True, job_utm__lt=utc_milliseconds()).exclude(job_utm=0): print(recipe.id, recipe.name, recipe.get_days()) print('---') for recipe in Recipe.objects.filter( active=True, worker_utm__gte=utc_milliseconds() - JOB_LOOKBACK_MS): print(recipe.id, recipe.name, recipe.worker_uid)
def setUp(self): self.account = account_create() self.project = project_create() now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles') today_tz = now_tz.strftime('%a') yesterday_tz = (now_tz - timedelta(hours=24)).strftime('%a') tomorrow_tz = (now_tz + timedelta(hours=24)).strftime('%a') self.recipe_today = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_TODAY', active=True, week=json.dumps([today_tz]), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([ { "tag": "hello", "values": {}, "sequence": 1 }, ]), worker_uid="OTHER_WORKER", worker_utm=utc_milliseconds() - ( JOB_LOOKBACK_MS * 10 ) # important for test ( jobs older than this get job_done flag reset ) ) self.recipe_not_today = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_NOT_TODAY', active=True, week=json.dumps([yesterday_tz, tomorrow_tz]), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([ { "tag": "hello", "values": {}, "sequence": 1 }, ]), worker_uid="OTHER_WORKER", worker_utm=utc_milliseconds() - ( JOB_LOOKBACK_MS * 10 ) # important for test ( jobs older than this get job_done flag reset ) )
def setUp(self): self.account = account_create() self.project = project_create() self.recipe = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_MANUAL', active = True, manual = True, week = [], hour = [], timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "manual", "values": {}, "sequence": 1 }, ]), ) now_tz = utc_to_timezone(datetime.utcnow(), self.recipe.timezone) self.recipe_done = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_MANUAL', active = True, manual = True, week = [], hour = [], timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "manual", "values": {}, "sequence": 1 }, ]), job_done = True, job_status = json.dumps({ "day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], "date_tz": str(now_tz.date()), "force": True, "tasks": [ { "instance": 1, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow()), "script": "manual", "hour": 0, "stdout":"", "stderr": "", "done": True } ] }), worker_uid = 'TEST_WORKER', worker_utm = utc_milliseconds() - (JOB_LOOKBACK_MS * 10) # important for test ( jobs older than this get job_done flag reset ) )
def autoscale(request): scale = { 'jobs': 0, 'workers': { 'jobs': settings.WORKER_JOBS, 'max': settings.WORKER_MAX, 'existing': 0, 'required': 0 } } # get task and worker list scale['jobs'] = Recipe.objects.filter( active=True, job_utm__lt=utc_milliseconds()).exclude(job_utm=0).count() scale['workers']['existing'] = 3 if request == 'TEST' else sum( 1 for instance in group_instances_list(('PROVISIONING', 'STAGING', 'RUNNING'))) scale['workers']['required'] = min( settings.WORKER_MAX, math.ceil(scale['jobs'] / scale['workers']['jobs'])) if request != 'TEST' and scale['workers']['required'] > scale['workers'][ 'existing']: group_instances_resize(scale['workers']['required']) # log the scaling operation log_manager_scale(scale) return JsonResponse(scale)
def setUp(self): self.account = account_create() self.project = project_create() self.recipe = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_MANUAL', active = True, manual = True, week = [], hour = [], timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "manual", "values": {}, "sequence": 1 }, ]), ) self.recipe.update() now_tz = utc_to_timezone(datetime.utcnow(), self.recipe.timezone) self.recipe_done = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_MANUAL', active = True, manual = True, week = [], hour = [], timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "manual", "values": {}, "sequence": 1 }, ]), job_status = json.dumps({ "date_tz": str(now_tz.date()), "tasks": [ { "instance": 1, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow()), "script": "manual", "hour": 0, "stdout":"", "stderr": "", "done": True } ] }), worker_uid = 'TEST_WORKER', worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE ) self.recipe_done.update()
def assertRecipeNotDone(cls, recipe): recipe.refresh_from_db() status = recipe.get_status() job_time = datetime.utcfromtimestamp(int(recipe.job_utm/1000)) cls.assertFalse(all(task['done'] == True for task in status['tasks'])) cls.assertLessEqual(recipe.job_utm, utc_milliseconds()) cls.assertEqual(job_time.minute, 0)
def setUp(self): self.account = account_create() self.project = project_create() now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles') self.job_done = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_DONE', active = True, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 }, ]), job_status = json.dumps({ "day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], "date_tz": str(now_tz.date()), "force": True, "tasks": [ { "instance": 1, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow()), "script": "hello", "hour": now_tz.hour, "stdout":"", "stderr": "", "done": True }, { "instance": 2, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow()), "script": "hello", "hour": now_tz.hour, "stdout":"", "stderr": "", "done": True } ] }), job_done = False, worker_uid = "SAMPLE_WORKER", worker_utm = utc_milliseconds() - (JOB_LOOKBACK_MS * 2) ) self.RECIPE_DONE = self.job_done.uid()
def test_time(self): utc_now = datetime.utcnow().replace(second=0, microsecond=0) tz_now = datetime.now(tz=pytz.timezone(self.recipe.timezone)).replace(second=0, microsecond=0) utm = utc_milliseconds(utc_now) tz1 = utc_milliseconds_to_timezone(utm, self.recipe.timezone) tz2 = utc_to_timezone(utc_now, self.recipe.timezone) tz3 = timezone_to_utc(tz_now) self.assertEqual(tz1, tz2) self.assertEqual(tz3, utc_now)
def worker_pull(worker_uid, jobs=1): '''Atomic reservation of worker in jobs. Args: - worker_uid ( string ) - identifies a unique worker, must be same for every call from same worker. - jobs ( integer ) - number of jobs to pull ''' jobs_all = [] jobs_new = [] worker_utm = utc_milliseconds() worker_lookback = worker_utm - JOB_LOOKBACK_MS worker_recheck = worker_utm - JOB_RECHECK_MS if jobs: with transaction.atomic(): # every half hour put jobs back in rotation so worker can trigger get_status logic, triggers status at 24 hour mark Recipe.objects.filter( active=True, manual=False, worker_utm__lt=worker_recheck).select_for_update( skip_locked=True).update(job_done=False) # find recipes that are available but have not been pinged recently from all workers ( pulls from pool ) where = Recipe.objects.filter( job_done=False, active=True, worker_utm__lt=worker_lookback, ).select_for_update( skip_locked=True).order_by('worker_utm').values_list( 'id', flat=True)[:jobs] # mark those recipes as belonging to this worker Recipe.objects.filter(id__in=where).update(worker_uid=worker_uid, worker_utm=worker_utm) # find all recipes that belong to this worker and check if they have new tasks for job in Recipe.objects.filter(job_done=False, active=True, worker_uid=worker_uid): jobs_all.append(job.id) task = job.get_task() # also checks if job is done if job.worker_utm == worker_utm and task: # jobs with current timestamp are new ( odds of a ping matching this worker_utm? ), isolate evens and odds? jobs_new.append(task) return jobs_all, jobs_new
def worker_ping(worker_uid, recipe_uids): # update recipes that belong to this worker if recipe_uids: Recipe.objects.filter( worker_uid=worker_uid, id__in=recipe_uids).update(worker_utm=utc_milliseconds())
def setUp(self): self.account = account_create() self.project = project_create() self.job_new = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_NEW', active=True, week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([ { "tag": "hello", "values": { "say_first": "Hi Once", "say_second": "Hi Twice", "sleep": 0 }, "sequence": 1 }, ]), ) self.RECIPE_NEW = self.job_new.uid() self.job_expired = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_EXPIRED', active=True, week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([ { "tag": "hello", "values": { "say_first": "Hi Once", "say_second": "Hi Twice", "sleep": 0 }, "sequence": 1 }, ]), job_done=False, worker_uid="SAMPLE_WORKER", worker_utm=utc_milliseconds() - (JOB_LOOKBACK_MS * 2)) self.RECIPE_EXPIRED = self.job_expired.uid() self.job_running = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_RUNNING', active=True, week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([{ "tag": "hello", "values": { "say_first": "Hi Once", "say_second": "Hi Twice", "sleep": 0 }, "sequence": 1 }]), job_done=False, worker_uid="OTHER_WORKER", worker_utm=utc_milliseconds()) self.RECIPE_RUNNING = self.job_running.uid() self.job_paused = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_PAUSED', active=False, week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([{ "tag": "hello", "values": { "say_first": "Hi Once", "say_second": "Hi Twice", "sleep": 0 }, "sequence": 1 }]), job_done=False, worker_uid="OTHER_WORKER", worker_utm=utc_milliseconds() - (JOB_LOOKBACK_MS * 10)) self.RECIPE_PAUSED = self.job_paused.uid() # paused so its not part of the normal flow ( unpause to use in test ) self.job_error = Recipe.objects.create( account=self.account, project=self.project, name='RECIPE_ERROR', active=False, week=json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour=json.dumps([0]), timezone='America/Los_Angeles', tasks=json.dumps([ { "tag": "hello", "values": { "say_first": "Hi Once", "say_second": "Hi Twice", "sleep": 0, "errror": "An error is triggered." }, "sequence": 1 }, ]), job_done=False, worker_uid="", worker_utm=0) self.RECIPE_ERROR = self.job_error.uid()
def test_job_utm(hour = 0, days_offset = 0): utc = datetime.utcnow().replace(hour=hour, minute=0, second=0, microsecond=0) if days_offset: utc += timedelta(days=days_offset) return utc_milliseconds(utc)
def setUp(self): self.account = account_create() self.project = project_create() now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles') status = { "date_tz":str(now_tz.date()), "tasks": [ { "instance": 1, "order": 0, "event": "JOB_PENDING", "utc":str(datetime.utcnow()), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": False }, { "instance": 2, "order": 2, "event": "JOB_PENDING", "utc":str(datetime.utcnow()), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": False }, ] } self.job_new = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_NEW', active = True, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 }, ]), ) self.RECIPE_NEW = self.job_new.uid() self.job_queued = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_QUEUED', active = True, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 }, ]), job_status = json.dumps(status.copy()), job_utm = test_job_utm() ) self.RECIPE_QUEUED = self.job_queued.uid() self.job_expired = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_EXPIRED', active = True, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 }, ]), job_status = json.dumps(status.copy()), job_utm = test_job_utm(), worker_uid = "SAMPLE_WORKER", worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE ) self.RECIPE_EXPIRED = self.job_expired.uid() self.job_running = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_RUNNING', active = True, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 } ]), job_status = json.dumps(status.copy()), job_utm = test_job_utm(), worker_uid = "OTHER_WORKER", worker_utm = utc_milliseconds() ) self.RECIPE_RUNNING = self.job_running.uid() self.job_paused = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_PAUSED', active = False, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0}, "sequence": 1 } ]), job_status = json.dumps(status.copy()), job_utm = test_job_utm(), worker_uid = "OTHER_WORKER", worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE ) self.RECIPE_PAUSED = self.job_paused.uid() # paused so its not part of the normal flow ( unpause to use in test ) self.job_error = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_ERROR', active = False, week = json.dumps(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {"say_first":"Hi Once", "say_second":"Hi Twice", "sleep":0, "errror":"An error is triggered."}, "sequence": 1 }, ]), job_status = json.dumps(status.copy()), job_utm = test_job_utm(), worker_uid = "", worker_utm = 0 ) self.RECIPE_ERROR = self.job_error.uid()
def setUp(self): self.account = account_create() self.project = project_create() now_tz = utc_to_timezone(datetime.utcnow(), 'America/Los_Angeles') today_tz = now_tz.strftime('%a') yesterday_tz = (now_tz - timedelta(days=1)).strftime('%a') tomorrow_tz = (now_tz + timedelta(days=1)).strftime('%a') self.recipe_today = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_TODAY', active = True, week = json.dumps([today_tz]), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {}, "sequence": 1 }, ]), job_status = json.dumps({ "date_tz":str(now_tz.date()), "tasks": [ { "instance": 1, "order": 0, "event": "JOB_PENDING", "utc":str(datetime.utcnow()), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": False }, { "instance": 2, "order": 0, "event": "JOB_PENDING", "utc":str(datetime.utcnow()), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": False } ] }), worker_uid = "OTHER_WORKER", worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE, job_utm = test_job_utm() ) self.recipe_not_today = Recipe.objects.create( account = self.account, project = self.project, name = 'RECIPE_NOT_TODAY', active = True, week = json.dumps([yesterday_tz, tomorrow_tz]), hour = json.dumps([0]), timezone = 'America/Los_Angeles', tasks = json.dumps([ { "tag": "hello", "values": {}, "sequence": 1 }, ]), job_status = json.dumps({ "date_tz":str(now_tz.date() - timedelta(days=1)), "tasks": [ { "instance": 1, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow() - timedelta(days=1)), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": True }, { "instance": 2, "order": 0, "event": "JOB_END", "utc":str(datetime.utcnow() - timedelta(days=1)), "script": "hello", "hour": 0, "stdout":"", "stderr": "", "done": True } ] }), worker_uid = "OTHER_WORKER", worker_utm = utc_milliseconds() - WORKER_LOOKBACK_EXPIRE, job_utm = test_job_utm(0, 1) )