def reset_match(request): group = Group.objects.get(name='Scorekeepers') if group not in request.user.groups.all(): raise Http404 ss = ScoringSystem.objects.all()[0] match = ss.current_match timer = (match.actual_start + datetime.timedelta(seconds=151)) - tz_aware_now() if timer.days >= 0 and ss.task_id: task = abort_match.delay() revoke(ss.task_id, terminate=True) from celery.task.control import discard_all discard_all() else: prematch_lighting() match.reset() ScoringDevice.objects.all().update(on_center=False) for tower_name in ['low_red', 'high_red', 'low_blue', 'high_blue']: setattr(match, 'scorer_' + tower_name, None) for sd in ScoringDevice.objects.filter(tower__isnull=False): setattr(match, 'scorer_' + sd.tower.name, sd.scorer) match.save() return scorekeeper(request)
def pytest_runtest_teardown(item): if not os.environ.get("USE_SNUBA", False): from sentry import tsdb # TODO(dcramer): this only works if this is the correct tsdb backend tsdb.flush() # XXX(dcramer): only works with DummyNewsletter from sentry import newsletter if hasattr(newsletter.backend, "clear"): newsletter.backend.clear() from sentry.utils.redis import clusters with clusters.get("default").all() as client: client.flushdb() from celery.task.control import discard_all discard_all() from sentry.models import OrganizationOption, ProjectOption, UserOption for model in (OrganizationOption, ProjectOption, UserOption): model.objects.clear_local_cache() Hub.main.bind_client(None)
def CelerydInit(**kwargs): from celery.task.control import discard_all discard_all() if not PIDRATOR_TESTING_WITHOUT_HARDWARE: import RPi.GPIO as GPIO GPIO.setmode(GPIO.BCM) GPIO.setwarnings(False)
def tearDown(self, *args, **kwargs): try: discard_all() except: pass super(DjangoFunctionalConservativeUnitTestMixin,self).tearDown(*args, **kwargs) call_command('flush', interactive=False) cache.clear()
def CelerydInit(**kwargs): from celery.task.control import discard_all if not PIMOSTAT_TESTING_WITHOUT_HARDWARE: import RPi.GPIO as GPIO GPIO.setmode(GPIO.BCM) GPIO.setwarnings(False) discard_all()
def pytest_runtest_teardown(item): from sentry.app import tsdb tsdb.flush() client = StrictRedis(db=9) client.flushdb() from celery.task.control import discard_all discard_all()
def tearDown(self, *args, **kwargs): try: discard_all() except: pass super(DjangoFunctionalConservativeUnitTestMixin, self).tearDown(*args, **kwargs) call_command('flush', interactive=False) cache.clear()
def discard_celery_jobs(): """ Discard all currently pending Celery jobs. @throws RabbitMQException if RabbitMQ cannot be contacted. @throws NoCeleryNodeException if no Celery nodes. """ try: discard_all() except socket.error as exc: raise RabbitMQException(exc)
def pytest_runtest_teardown(item): from sentry.app import tsdb tsdb.flush() from sentry.utils.redis import clusters with clusters.get('default').all() as client: client.flushdb() from celery.task.control import discard_all discard_all()
def pytest_runtest_teardown(item): for _, teardown in tasks: if teardown: teardown() from celery.task.control import discard_all discard_all() from sentry.models import OrganizationOption, ProjectOption, UserOption for model in (OrganizationOption, ProjectOption, UserOption): model.objects.clear_local_cache()
def pytest_runtest_teardown(item): from sentry import tsdb # TODO(dcramer): this only works if this is the correct tsdb backend tsdb.backend.flush() from sentry.utils.redis import clusters with clusters.get('default').all() as client: client.flushdb() from celery.task.control import discard_all discard_all()
def stop_all(request): discard_all() active_sims = Simulation.objects.filter(task_id__isnull=False).all() for simulation in active_sims: revoke(simulation.task_id, terminate=True) simulation.task_id = None simulation.save(update_fields=['task_id']) discard_all() return HttpResponse("stopped all")
def reset(args): """ Resets the queue and removes the population files. """ def confirm(prompt="Continue? [y/n] "): valid = {'yes': True, 'y': True, 'no': False, 'n': False} choice = raw_input(prompt).lower() if choice in valid: return valid[choice] return confirm(prompt) if not args.force: print "Resetting will permanently remove population and Queue!" print "Removing from: %s" % args.dirname if not confirm(): return "Exiting without deleting anything." files = 0 for name in os.listdir(args.dirname): path = os.path.join(args.dirname, name) try: if os.path.isfile(path): os.remove(path) files += 1 except Exception as e: print "Error removing %s: %s" % (path, str(e)) queue = discard_all() return "%i files removed and %i tasks discarded" % (files, queue)
def reset(args): """ Resets the queue and removes the population files. """ def confirm(prompt="Continue? [y/n] "): valid = {'yes': True, 'y': True, 'no': False, 'n':False} choice = raw_input(prompt).lower() if choice in valid: return valid[choice] return confirm(prompt) if not args.force: print "Resetting will permanently remove population and Queue!" print "Removing from: %s" % args.dirname if not confirm(): return "Exiting without deleting anything." files = 0 for name in os.listdir(args.dirname): path = os.path.join(args.dirname, name) try: if os.path.isfile(path): os.remove(path) files += 1 except Exception as e: print "Error removing %s: %s" % (path, str(e)) queue = discard_all() return "%i files removed and %i tasks discarded" % (files, queue)
def pytest_runtest_teardown(item): from sentry import tsdb # TODO(dcramer): this only works if this is the correct tsdb backend tsdb.flush() # XXX(dcramer): only works with DummyNewsletter from sentry import newsletter if hasattr(newsletter.backend, 'clear'): newsletter.backend.clear() from sentry.utils.redis import clusters with clusters.get('default').all() as client: client.flushdb() from celery.task.control import discard_all discard_all()
def setUpModule(): # pylint: disable=global-statement global tmpdir global runpath global logpath global old_btw_env_suppress_message tmpdir = tempfile.mkdtemp(prefix="btw-test-btwworker") runpath = os.path.join(tmpdir, "run") logpath = os.path.join(tmpdir, "log") os.mkdir(runpath) os.mkdir(logpath) old_btw_env_suppress_message = os.environ.get("BTW_ENV_SUPPRESS_MESSAGE") os.environ["BTW_ENV_SUPPRESS_MESSAGE"] = "1" # We purge all tasks before running the tests here so that the workers # do not attempt to do anything. from celery.task.control import discard_all discard_all()
def pytest_runtest_teardown(item): from sentry import tsdb # TODO(dcramer): this only works if this is the correct tsdb backend tsdb.flush() # XXX(dcramer): only works with DummyNewsletter from sentry import newsletter if hasattr(newsletter.backend, 'clear'): newsletter.backend.clear() from sentry.utils.redis import clusters with clusters.get('default').all() as client: client.flushdb() from celery.task.control import discard_all discard_all() from sentry.models import OrganizationOption, ProjectOption, UserOption for model in (OrganizationOption, ProjectOption, UserOption): model.objects.clear_local_cache()
def clear_twitter(): control = Control(app) #control.cancel_consumer('streamapi') # queue name, must probably be specified once per queue, but my app uses a single queue inspect = control.inspect() if True: active = inspect.active() print active running_jobs = [] if active != None: for key, value in active.items(): running_jobs.extend(value) if len(running_jobs) > 0: print("{} jobs running: {}".format(len(running_jobs), ", ".join(job["name"] for job in running_jobs))) for job in running_jobs: #if job['name'] == 'fetch': control.revoke(job["id"],terminate=True) discard_all() #time.sleep(10) else: print("No running jobs") app2 = state.State() cont = Control() i = Inspect() query = app2.tasks_by_type('fetch') print 'query ' ,query for uuid, task in query: #cont.revoke(uuid, terminate=True) print uuid, task #cont.purge() a = discard_all() print a
def master(hb_config, rip_config, in_path, out_path): # discard possible jobs from before try: # TODO: Move this call to funtion in task of job module?! discard_all() except OSError: print('Could not connect to host') sys.exit(-1) disc_list = [] for root, dirs, files in os.walk(in_path): if dirs: print('Subdirs currently not supported!') break for f in files: disc_list.append(Disc(os.path.join(root, f), f)) available_job_queue = queue.Queue() ready_job_queue = queue.Queue() check_thread = threading.Thread(target=master_check_tasks_thread, args=(available_job_queue, ready_job_queue)) check_thread.daemon = True check_thread.start() teardown_thread = threading.Thread(target=master_teardown_thread, args=(ready_job_queue,)) teardown_thread.daemon = True teardown_thread.start() for d in disc_list: logger.info('creating job {}'.format(d)) job = Job(d, rip_config, hb_config, in_path, out_path) try: job.run(handbrake_task.delay) available_job_queue.put(job) except ConnectionResetError: print('Could not run job. Maybe broker credentials invalid?!') available_job_queue.join() ready_job_queue.join()
def reset_match(request): group = Group.objects.get(name='Scorekeepers') if group not in request.user.groups.all(): raise Http404 ss = ScoringSystem.objects.all()[0] match = ss.current_match timer = (match.actual_start + datetime.timedelta(seconds=151)) - tz_aware_now() if timer.days >= 0 and ss.task_id: task = abort_match.delay() revoke(ss.task_id, terminate=True) from celery.task.control import discard_all discard_all() else: prematch_lighting() match.reset() ScoringDevice.objects.all().update(on_center=False) for tower_name in ['low_red', 'high_red', 'low_blue', 'high_blue']: setattr(match, 'scorer_'+tower_name, None) for sd in ScoringDevice.objects.filter(tower__isnull=False): setattr(match, 'scorer_'+sd.tower.name, sd.scorer) match.save() return scorekeeper(request)
def cancel_all_tasks(): discard_all()
def create_rabbitmq(self): self.rabbitmq_process = Process(target=self.run_rabbitmq) self.rabbitmq_process.start() time.sleep(1) discard_all()
def handle(self, *args, **options): discard_all()
def handle(self, *args, **options): discarded = discard_all() self.stdout.write("Discarded %s celery tasks" % discarded)
def celery_purge(): discard_all()
t = time.mktime(time.strptime(time_placed, ts_format)) trade['timestamp'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t)) trade['unixtime'] = time.strftime("%s", time.localtime(t)) # calculate the trade amount bought in euros if 'EUR' in trade['currencyFrom']: amount_buy = trade['currencyFrom'] elif 'EUR' in trade['currencyTo']: amount_buy = trade['currencyTo'] else: year_rates = g.rates['20' + trade['timePlaced'][7:9]]['rates'] eur_amount = trade['amountSell'] / year_rates[trade['currencyFrom']] amount_buy = eur_amount * year_rates[trade['currencyTo']] amount_buy = round(eur_amount, g.currencies[trade['currencyTo']]['decimals']) trade['amountEur'] = amount_buy trade['currencyPair'] = "{0}/{1}".format(trade['currencyFrom'],trade['currencyTo']) trade['originatingCountryName'] = g.countries[trade['originatingCountry']]['name'] data = r.db('cf').table('processed').insert(trade).run() return data else: return True else: return False if __name__ == '__main__': #celeryctl purge from celery.task.control import discard_all discard_all() print(process_trade('1e730047-520e-4c9b-a08e-d2fe7c9f198c'))
def cleanup(): p.terminate() b.terminate() discard_all()
def test_discard_all(self): control.discard_all()
def cleanup(self): # clean up queues n = control.discard_all() if n: print 'LoadGenerator.cleanup() purged {0} tasks.'.format(n)
# close() or join() equivalent for celery jobs (python)? from celery.task.control import discard_all discard_all()
def cancel_all_view(self, request): n = discard_all() self.message_user(request, 'Discarded %d tasks' % n) return self.changelist_view(request)
def cache_list(key=None): ''' @method GET: Returns this list of flask cache keys and their time to live (TTL) :return: JSON @method DELETE: Send a delete request passing in the name of the cache key that needs to be deleted. :return: JSON ''' if request.method == 'GET': # setup the container for the list of current cache items flask_cache = [] # issue the command to get the list from redis pipe_output = subprocess.Popen(['redis-cli', 'keys', '*'], stdout=subprocess.PIPE) output, err = pipe_output.communicate() # the output is a string, so lets load it into the array flask_cache = output.split('\n') # we don't want to provide system keys, remove non "flask_cache" temp_list = [] for cache_key in flask_cache: if 'flask_cache' in cache_key: temp_list.append({'key': cache_key, 'name': cache_key.split('_')[2] }) # assign the list to the main bin flask_cache = temp_list # lets get the TTL of each of the keys for cache_key in flask_cache: pipe_output = subprocess.Popen(['redis-cli', 'TTL', cache_key['key']], stdout=subprocess.PIPE) output, err = pipe_output.communicate() cache_key['TTL'] = output.split('\n')[0] # clear out this for garbage collection of flask_cache ref temp_list = None return jsonify({'results': flask_cache}) elif request.method == 'DELETE': try: # grab the key, lets give it a more readable reference redis_key = key # perform the delete operation on the target host's redis server pipe_output = subprocess.Popen(['redis-cli', 'del', redis_key], stdout=subprocess.PIPE) output, err = pipe_output.communicate() # we'll discard all previous celery jobs, so new ones can be queued up. # Note, this does NOT issue a cache reload. It simply removes any # backloged tasks. Keep it tight, like a tiger. if output == 1: discard_all() return jsonify({'results': int(output)}), 200 except Exception as e: return jsonify({'error': 'Exception in cache delete. %s' % e}), 500
def reset_test_database(request): # pragma: no cover """ Loads a database with fake data for testing. """ # Clear Celery tasks discard_all() # Delete Submissions for root, dirs, files in os.walk('submissions'): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) # Delete current database Submission.objects.all().delete() SubmissionFile.objects.all().delete() Course.objects.all().delete() Assignment.objects.all().delete() AssignedReview.objects.all().delete() Comment.objects.all().delete() CommentRange.objects.all().delete() EmailPreferences.objects.all().delete() Question.objects.all().delete() Response.objects.all().delete() ### FOR MAURICIOS INDIVIDUAL PROJECT Faq.objects.all().delete() ################ now = timezone.now() just_before = timezone.now() - datetime.timedelta(seconds=10) before = timezone.now() - datetime.timedelta(days=1) long_before = timezone.now() - datetime.timedelta(days=30) just_after = timezone.now() + datetime.timedelta(seconds=30) after = timezone.now() + datetime.timedelta(days=1) long_after = timezone.now() + datetime.timedelta(days=30) #Deal with users if len(User.objects.all()) <= 1: user1 = User(username="******") user1.set_password('user1') user1.save() user2 = User(username="******") user2.set_password('user2') user2.save() user3 = User(username="******") user3.set_password('user3') user3.save() user4 = User(username="******") user4.set_password('user4') user4.save() else: user1 = User.objects.filter(username="******")[0] user2 = User.objects.filter(username="******")[0] user3 = User.objects.filter(username="******")[0] user4 = User.objects.filter(username="******")[0] user1prefs = EmailPreferences(user=user1, on_day_before_due=True, on_due_date=True) user1prefs.save() user2prefs = EmailPreferences(user=user2) user2prefs.save() user3prefs = EmailPreferences(user=user3) user3prefs.save() user4prefs = EmailPreferences(user=user4) user4prefs.save() course = Course(course_name="Test Course", course_code="TEST1000", course_id="201402TEST1000", year=2014, semester='2', institution="UQ") course.save() a1 = Assignment( assignment_id="ass01", name="Assignment 1", create_date=long_before, modified_date=long_before, open_date=long_before, due_date=just_before, description="This is assignment 1. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", description_raw="This is assignment 1. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", allow_multiple_uploads=True, allow_help_centre=True, number_of_peer_reviews=3, review_open_date=just_before, review_due_date=after, weighting=20, test_zip="", dockerfile="", docker_command="", test_required=False ) a1.save() a1.set_async() print "a1 created" a2 = Assignment( assignment_id="ass02", name="Assignment 2", create_date=before, modified_date=before, open_date=before, due_date=just_after, description="This is assignment 2. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", description_raw="This is assignment 2. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", allow_multiple_uploads=True, allow_help_centre=True, number_of_peer_reviews=3, review_open_date=just_after, review_due_date=long_after, weighting=20, test_zip="", dockerfile="", docker_command="", test_required=False ) a2.save() a2.set_async() print "a2 created" a3 = Assignment( assignment_id="ass03", name="Assignment 3", create_date=now, modified_date=now, open_date=just_after, due_date=long_after, description="This is assignment 3. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", description_raw="This is assignment 3. Bacon ipsum dolor sit amet short loin jowl swine, drumstick hamburger meatball prosciutto frankfurter chuck. Shank sausage doner meatball shankle flank, t-bone venison turkey jerky bacon strip steak ribeye chicken pastrami. Capicola ground round corned beef turducken frankfurter. Jowl landjaeger bacon, sausage frankfurter meatball rump bresaola strip steak fatback short loin jerky pancetta turducken.", allow_multiple_uploads=True, allow_help_centre=True, number_of_peer_reviews=3, review_open_date=long_after, review_due_date=long_after, weighting=20, test_zip="testing/test.zip", dockerfile="testing/Dockerfile", docker_command="python /opt/testing/test.py", test_required=False ) a3.save() a3.set_async() print "a3 created" ## Prev assignment w/ review submission0 = Submission( user = user1, assignment = a1, upload_date = before, upload_path = "codereview/test_submission/0/", has_been_submitted = True, unviewed_reviews = 1 ) submission0.save() submission0b = Submission( user = user2, assignment = a1, upload_date = before, upload_path = "codereview/test_submission/0/", has_been_submitted = True, unviewed_reviews = 1 ) submission0b.save() sf0 = SubmissionFile(submission=submission0, file_path="test.py") sf0.save() sf0b = SubmissionFile(submission=submission0b, file_path="test.py") sf0b.save() assigned_review = AssignedReview(assigned_user=user2, assigned_submission=submission0, has_been_reviewed=True) assigned_review.save() assigned_review2 = AssignedReview(assigned_user=user1, assigned_submission=submission0b, has_been_reviewed=False) assigned_review2.save() comment = Comment(commenter=user2, commented_file=sf0, comment="You probably shouldn't hard code the IP address.", selected_text='mpd_client.connect("192.168.1.4", 6600)') comment.save() comment_range = CommentRange(comment=comment, start='', end='',startOffset=195, endOffset=234) comment_range.save() assigned_review3 = AssignedReview(assigned_user=user3, assigned_submission=submission0, has_been_reviewed=True) assigned_review3.save() comment2 = Comment(commenter=user3, commented_file=sf0, comment="This function doesn't cover all possibilities and will return None. You should probably find a better way to implement it.", selected_text='def parse_command(text):') comment2.save() comment_range2 = CommentRange(comment=comment2, start='', end='',startOffset=521, endOffset=725) comment_range2.save() submission1 = Submission( user = user2, assignment = a2, upload_date = just_before, upload_path = "codereview/test_submission/1/", has_been_submitted = True, unviewed_reviews = 0 ) submission1.save() sf1 = SubmissionFile(submission=submission1, file_path="email.py") sf1.save() submission2 = Submission( user = user3, assignment = a2, upload_date = just_before, upload_path = "codereview/test_submission/2/", has_been_submitted = True, unviewed_reviews = 0 ) submission2.save() sf2 = SubmissionFile(submission=submission2, file_path="email.py") sf2.save() submission3 = Submission( user = user4, assignment = a2, upload_date = just_before, upload_path = "codereview/test_submission/3/", has_been_submitted = True, unviewed_reviews = 0 ) submission3.save() sf3 = SubmissionFile(submission=submission3, file_path="email.py") sf3.save() question1 = Question( user = user4, title = "Why is the sky blue?", text = "Tell me why.", text_raw = "Tell me why.", create_date = long_before, modified_date = just_before, ) question1.save() question2 = Question( user = user4, title = "Why did the chicken cross the road?", text = "Bacon ipsum dolor sit amet short loin jowl swine.", text_raw = "Bacon ipsum dolor sit amet short loin jowl swine.", create_date = long_before, modified_date = long_before, ) question2.save() answer1 = Response( user = user2, question = question2, text = "Just because.", text_raw = "Just because.", create_date = before, modified_date = before, ) answer1.save() answer2 = Response( user = user2, question = question2, text = "Egg.", text_raw = "Egg.", create_date = before, modified_date = before, ) answer2.save() ## FOR MAURICIOS INDIVIDUAL PROJECT faq1 = Faq( title="This is a frequently asked question.", text = "Stop asking it dummy!", text_raw = "Stop asking it dummy!", ) faq1.save() ####### return redirect('index')