def real_main(release_url=None, casefile_path=None, upload_build_id=None, upload_release_name=None): """Runs diff_my_images.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() json_write(FLAGS.casefile_path) tests_json_path = FLAGS.casefile_path + "/tests_case.json" data = open(tests_json_path).read() tests = load_tests(data) item = DiffMyImages(release_url, tests, upload_build_id, upload_release_name, heartbeat=workers.PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_one() coordinator.stop() coordinator.join()
def testNoDifferences(self): """Tests crawling the site end-to-end.""" @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world!' site_diff.real_main( start_url='http://%s:%d/' % test.server_address, output_dir=self.reference_dir, coordinator=self.coordinator) self.coordinator = workers.get_coordinator() site_diff.real_main( start_url='http://%s:%d/' % test.server_address, output_dir=self.output_dir, reference_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() self.assertTrue(exists(join(self.reference_dir, '__run.log'))) self.assertTrue(exists(join(self.reference_dir, '__run.png'))) self.assertTrue(exists(join(self.reference_dir, '__config.js'))) self.assertTrue(exists(join(self.reference_dir, 'url_paths.txt'))) self.assertTrue(exists(join(self.output_dir, '__run.log'))) self.assertTrue(exists(join(self.output_dir, '__run.png'))) self.assertTrue(exists(join(self.output_dir, '__ref.log'))) self.assertTrue(exists(join(self.output_dir, '__ref.png'))) self.assertFalse(exists(join(self.output_dir, '__diff.png'))) # No diff self.assertTrue(exists(join(self.output_dir, '__diff.log'))) self.assertTrue(exists(join(self.output_dir, '__config.js'))) self.assertTrue(exists(join(self.output_dir, 'url_paths.txt')))
def testNoDifferences(self): """Tests crawling the site end-to-end.""" @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world!' site_diff.real_main(start_url='http://%s:%d/' % test.server_address, output_dir=self.reference_dir, coordinator=self.coordinator) self.coordinator = workers.get_coordinator() site_diff.real_main(start_url='http://%s:%d/' % test.server_address, output_dir=self.output_dir, reference_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() self.assertTrue(exists(join(self.reference_dir, '__run.log'))) self.assertTrue(exists(join(self.reference_dir, '__run.png'))) self.assertTrue(exists(join(self.reference_dir, '__config.js'))) self.assertTrue(exists(join(self.reference_dir, 'url_paths.txt'))) self.assertTrue(exists(join(self.output_dir, '__run.log'))) self.assertTrue(exists(join(self.output_dir, '__run.png'))) self.assertTrue(exists(join(self.output_dir, '__ref.log'))) self.assertTrue(exists(join(self.output_dir, '__ref.png'))) self.assertFalse(exists(join(self.output_dir, '__diff.png'))) # No diff self.assertTrue(exists(join(self.output_dir, '__diff.log'))) self.assertTrue(exists(join(self.output_dir, '__config.js'))) self.assertTrue(exists(join(self.output_dir, 'url_paths.txt')))
def setUp(self): """Sets up the test harness.""" FLAGS.queue_idle_poll_seconds = 0.01 FLAGS.queue_busy_poll_seconds = 0.01 self.coordinator = workers.get_coordinator() fetch_worker.register(self.coordinator) timer_worker.register(self.coordinator) self.coordinator.start()
def setUp(self): """Sets up the test harness.""" FLAGS.fetch_frequency = 100 FLAGS.polltime = 0.01 self.test_dir = tempfile.mkdtemp('site_diff_test') self.output_dir = join(self.test_dir, 'output') self.reference_dir = join(self.test_dir, 'reference') self.coordinator = workers.get_coordinator()
def run_workers(): coordinator = workers.get_coordinator() capture_worker.register(coordinator) fetch_worker.register(coordinator) pdiff_worker.register(coordinator) timer_worker.register(coordinator) coordinator.start() logging.info('Workers started') return coordinator
def setUp(self): """Sets up the test harness.""" FLAGS.fetch_frequency = 100 FLAGS.polltime = 0.01 self.coordinator = workers.get_coordinator() self.echo_queue = Queue.Queue() self.coordinator.register(EchoItem, self.echo_queue) self.coordinator.register(FireAndForgetEchoItem, self.echo_queue) self.coordinator.worker_threads.append(EchoThread(self.echo_queue, self.coordinator.input_queue)) self.timer_queue = Queue.Queue() self.coordinator.register(timer_worker.TimerItem, self.timer_queue) self.coordinator.worker_threads.append(timer_worker.TimerThread(self.timer_queue, self.coordinator.input_queue)) self.coordinator.start()
def real_main(start_url=None, ignore_prefixes=None, upload_build_id=None, upload_release_name=None): """Runs the site_diff.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = SiteDiff( start_url=start_url, ignore_prefixes=ignore_prefixes, upload_build_id=upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow, ) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def real_main(start_url=None, ignore_prefixes=None, upload_build_id=None, upload_release_name=None): """Runs the site_diff.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = SiteDiff(start_url=start_url, ignore_prefixes=ignore_prefixes, upload_build_id=upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def real_main(new_url=None, baseline_url=None, upload_build_id=None, upload_release_name=None): """Runs the ur_pair_diff.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = UrlPairDiff(new_url, baseline_url, upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def real_main(new_url=None, baseline_url=None, upload_build_id=None, upload_release_name=None): """Runs the ur_pair_diff.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = UrlPairDiff( new_url, baseline_url, upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def real_main(start_url=None, ignore_prefixes=None, upload_build_id=None, upload_release_name=None, coordinator=None): """Runs the site_diff.""" if not coordinator: coordinator = workers.get_coordinator() coordinator.start() item = SiteDiff( start_url=start_url, ignore_prefixes=ignore_prefixes, upload_build_id=upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_until_interrupted()
def setUp(self): """Sets up the test harness.""" FLAGS.fetch_frequency = 100 FLAGS.polltime = 0.01 self.coordinator = workers.get_coordinator() self.echo_queue = Queue.Queue() self.coordinator.register(EchoItem, self.echo_queue) self.coordinator.register(FireAndForgetEchoItem, self.echo_queue) self.coordinator.worker_threads.append( EchoThread(self.echo_queue, self.coordinator.input_queue)) self.timer_queue = Queue.Queue() self.coordinator.register(timer_worker.TimerItem, self.timer_queue) self.coordinator.worker_threads.append( timer_worker.TimerThread(self.timer_queue, self.coordinator.input_queue)) self.coordinator.start()
def real_main(new_url=None, baseline_url=None, upload_build_id=None, upload_release_name=None, coordinator=None): """Runs the ur_pair_diff.""" if not coordinator: coordinator = workers.get_coordinator() coordinator.start() item = UrlPairDiff( new_url, baseline_url, upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_until_interrupted()
def testOneDifference(self): """Tests when there is one found difference.""" @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world!' site_diff.real_main( start_url='http://%s:%d/' % test.server_address, output_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world a little different!' self.coordinator = workers.get_coordinator() site_diff.real_main( start_url='http://%s:%d/' % test.server_address, output_dir=self.output_dir, reference_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() self.assertTrue(exists(join(self.reference_dir, '__run.log'))) self.assertTrue(exists(join(self.reference_dir, '__run.png'))) self.assertTrue(exists(join(self.reference_dir, '__config.js'))) self.assertTrue(exists(join(self.reference_dir, 'url_paths.txt'))) self.assertTrue(exists(join(self.output_dir, '__run.log'))) self.assertTrue(exists(join(self.output_dir, '__run.png'))) self.assertTrue(exists(join(self.output_dir, '__ref.log'))) self.assertTrue(exists(join(self.output_dir, '__ref.png'))) self.assertTrue(exists(join(self.output_dir, '__diff.png'))) # Diff!! self.assertTrue(exists(join(self.output_dir, '__diff.log'))) self.assertTrue(exists(join(self.output_dir, '__config.js'))) self.assertTrue(exists(join(self.output_dir, 'url_paths.txt')))
def real_main(upload_build_id=None, upload_release_name=None, old_endpoint=None, new_endpoint=None, path='', prepend_old='', prepend_new='', run_name_depth=2): """Runs pair diffs between URL pairs in the given config file""" print "Gathering URLs from %s" % old_endpoint old_images = parse_urls(json_str=get_url(old_endpoint), json_path=path, url_append=prepend_old) print "Gathering URLs from %s" % new_endpoint new_images = parse_urls(json_str=get_url(new_endpoint), json_path=path, url_append=prepend_new) print 'Pairing images ...' data = map_urls(base_urls=old_images, new_urls=new_images) coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = UrlPairSetDiff( image_pairs=data['pairs'], upload_build_id=upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow, run_name_depth=run_name_depth) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def real_main(release_url=None, tests_json_path=None, upload_build_id=None, upload_release_name=None): """Runs diff_my_images.""" coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() data = open(FLAGS.tests_json_path).read() tests = load_tests(data) item = DiffMyImages( release_url, tests, upload_build_id, upload_release_name, heartbeat=workers.PrintWorkflow) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
def testOneDifference(self): """Tests when there is one found difference.""" @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world!' site_diff.real_main(start_url='http://%s:%d/' % test.server_address, output_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() @webserver def test(path): if path == '/': return 200, 'text/html', 'Hello world a little different!' self.coordinator = workers.get_coordinator() site_diff.real_main(start_url='http://%s:%d/' % test.server_address, output_dir=self.output_dir, reference_dir=self.reference_dir, coordinator=self.coordinator) test.shutdown() self.assertTrue(exists(join(self.reference_dir, '__run.log'))) self.assertTrue(exists(join(self.reference_dir, '__run.png'))) self.assertTrue(exists(join(self.reference_dir, '__config.js'))) self.assertTrue(exists(join(self.reference_dir, 'url_paths.txt'))) self.assertTrue(exists(join(self.output_dir, '__run.log'))) self.assertTrue(exists(join(self.output_dir, '__run.png'))) self.assertTrue(exists(join(self.output_dir, '__ref.log'))) self.assertTrue(exists(join(self.output_dir, '__ref.png'))) self.assertTrue(exists(join(self.output_dir, '__diff.png'))) # Diff!! self.assertTrue(exists(join(self.output_dir, '__diff.log'))) self.assertTrue(exists(join(self.output_dir, '__config.js'))) self.assertTrue(exists(join(self.output_dir, 'url_paths.txt')))
def real_main(upload_build_id=None, upload_release_name=None, old_endpoint=None, new_endpoint=None, path='', prepend_old='', prepend_new='', run_name_depth=2): """Runs pair diffs between URL pairs in the given config file""" print "Gathering URLs from %s" % old_endpoint old_images = parse_urls(json_str=get_url(old_endpoint), json_path=path, url_append=prepend_old) print "Gathering URLs from %s" % new_endpoint new_images = parse_urls(json_str=get_url(new_endpoint), json_path=path, url_append=prepend_new) print 'Pairing images ...' data = map_urls(base_urls=old_images, new_urls=new_images) coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() item = UrlPairSetDiff(image_pairs=data['pairs'], upload_build_id=upload_build_id, upload_release_name=upload_release_name, heartbeat=PrintWorkflow, run_name_depth=run_name_depth) item.root = True coordinator.input_queue.put(item) coordinator.wait_one()
config_dir = argv[2] assert os.path.isdir(config_dir), 'Expected directory, got %s' % config_dir utils.verify_binary('phantomjs_binary', ['--version']) utils.verify_binary('pdiff_compare_binary', ['--version']) utils.verify_binary('pdiff_composite_binary', ['--version']) assert os.path.exists(FLAGS.phantomjs_script) logging.basicConfig() logging.getLogger().addFilter(RepetitiveLogFilterer()) logging.getLogger().handlers[0].setFormatter(CompactExceptionLogger()) if FLAGS.verbose: logging.getLogger().setLevel(logging.DEBUG) coordinator = workers.get_coordinator() timer_worker.register(coordinator) global FAILED_TESTS FAILED_TESTS = 0 item = RunAllTestSuitesWorkflowItem(config_dir, mode) item.root = True coordinator.input_queue.put(item, mode) coordinator.start() coordinator.wait_one() coordinator.stop() coordinator.join() if mode == 'test': if FAILED_TESTS > 0:
def setUp(self): """Sets up the test harness.""" self.coordinator = workers.get_coordinator() self.build_id = create_build() self.release_name = uuid.uuid4().hex self.client = server.app.test_client()
def release_and_run(): passed_key = request.form.get('G5_DPXDT_API_KEY', default=None, type=str) if passed_key != os.environ['G5_DPXDT_API_KEY']: return flask.jsonify(error="invalid or missing API key") build = request.form.get('build', default=None, type=int) url = request.form.get('url', default=None, type=str) name = request.form.get('name', default=None, type=str) depth = request.form.get('depth', default=1, type=int) #name supercedes build if name: msg = "Build determined via name: %s. " % name bd = models.Build.query.filter_by(name=name).first() if not bd: bd = _create_build(name) msg += " Build did not exist, created it. " # return flask.jsonify(error="build by that name does not exist.") build = bd.id else: msg = "Build id taken from passed arg: %s. " % build #however we determined the build, make sure we have a url if not url: rel = models.Release.query.filter_by(build_id=build)\ .order_by(models.Release.created.desc()).first() if not rel: return flask.jsonify(error="no url provided and no previous releases to extrapolate from.") url = rel.url msg += "url determined via last release in the build: %s. " % url else: msg += "url taken from passed arg: %s. " % url utils.jsonify_assert(build, 'must supply a build or a name') utils.jsonify_assert(url, 'must supply a url or a name') FLAGS.crawl_depth = depth pull_inject_code() coordinator = workers.get_coordinator() fetch_worker.register(coordinator) coordinator.start() sd = SiteDiff( start_url=url, ignore_prefixes=None, upload_build_id=build, upload_release_name=url, heartbeat=workers.PrintWorkflow) sd.root = True coordinator.input_queue.put(sd) msg += "Job(s) started." return flask.jsonify( success=True, msg=msg, )
def setUp(self): """Sets up the test harness.""" FLAGS.fetch_frequency = 100 FLAGS.polltime = 0.01 self.coordinator = workers.get_coordinator()