def cleanup_index(request, es, index_name): def fin(): try: es.indices.delete(index_name) except NotFoundError: pass request.addfinalizer(fin)
def file_server6(request): time.sleep(0.1) file_server6 = FileServer("::1", 1544) file_server6.ip_external = 'fca5:95d6:bfde:d902:8951:276e:1111:a22c' # Fake external ip def listen(): ConnectionServer.start(file_server6) ConnectionServer.listen(file_server6) gevent.spawn(listen) # Wait for port opening for retry in range(10): time.sleep(0.1) # Port opening try: conn = file_server6.getConnection("::1", 1544) conn.close() break except Exception as err: print("FileServer6 startup error", Debug.formatException(err)) assert file_server6.running file_server6.ip_incoming = {} # Reset flood protection def stop(): file_server6.stop() request.addfinalizer(stop) return file_server6
def install_context(request, tmpdir, monkeypatch): """Fixture to set up temporary installation directory. """ # Save old values so we can restore them. new_cwd = tmpdir.mkdir('cwd') user_base = tmpdir.mkdir('user_base') user_site = tmpdir.mkdir('user_site') install_dir = tmpdir.mkdir('install_dir') def fin(): # undo the monkeypatch, particularly needed under # windows because of kept handle on cwd monkeypatch.undo() new_cwd.remove() user_base.remove() user_site.remove() install_dir.remove() request.addfinalizer(fin) # Change the environment and site settings to control where the # files are installed and ensure we do not overwrite anything. monkeypatch.chdir(new_cwd) monkeypatch.setattr(easy_install_pkg, '__file__', user_site.strpath) monkeypatch.setattr('site.USER_BASE', user_base.strpath) monkeypatch.setattr('site.USER_SITE', user_site.strpath) monkeypatch.setattr('sys.path', sys.path + [install_dir.strpath]) monkeypatch.setenv(str('PYTHONPATH'), str(os.path.pathsep.join(sys.path))) # Set up the command for performing the installation. dist = Distribution() cmd = easy_install(dist) cmd.install_dir = install_dir.strpath return cmd
def minikube(request, worker_id): def teardown(): if not k8s_skip_teardown: try: print("Removing minikube container ...") mk.container.remove(force=True, v=True) except: pass request.addfinalizer(teardown) k8s_version = "v" + request.param.lstrip("v") k8s_timeout = int(request.config.getoption("--k8s-timeout")) k8s_container = request.config.getoption("--k8s-container") k8s_skip_teardown = request.config.getoption("--k8s-skip-teardown") mk = Minikube() mk.worker_id = worker_id if k8s_container: mk.connect(k8s_container, k8s_timeout) k8s_skip_teardown = True elif worker_id == "master" or worker_id == "gw0": mk.deploy(k8s_version, k8s_timeout) if worker_id == "gw0": k8s_skip_teardown = True else: mk.connect("minikube", k8s_timeout, version=k8s_version) k8s_skip_teardown = True return mk
def cleanup_index(request, es, index_name): def fin(): try: es.indices.delete(index_name) except NotFoundError: pass request.addfinalizer(fin)
def stop(request): def fin(): if fixture: fixture.destroy() request.addfinalizer(fin) return fixture
def site(request): threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)] # Reset ratelimit RateLimit.queue_db = {} RateLimit.called_db = {} site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") # Always use original data assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in site.storage.getPath("") # Make sure we dont delete everything shutil.rmtree(site.storage.getPath(""), True) shutil.copytree(site.storage.getPath("") + "-original", site.storage.getPath("")) # Add to site manager SiteManager.site_manager.get("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") site.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net def cleanup(): site.delete() site.content_manager.contents.db.close("Test cleanup") site.content_manager.contents.db.timer_check_optional.kill() SiteManager.site_manager.sites.clear() db_path = "%s/content.db" % config.data_dir os.unlink(db_path) del ContentDb.content_dbs[db_path] gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before]) request.addfinalizer(cleanup) site.greenlet_manager.stopGreenlets() site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") # Create new Site object to load content.json files if not SiteManager.site_manager.sites: SiteManager.site_manager.sites = {} SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] = site site.settings["serving"] = True return site
def site_temp(request): threads_before = [ obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) ] with mock.patch("Config.config.data_dir", config.data_dir + "-temp"): site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") site_temp.settings["serving"] = True site_temp.announce = mock.MagicMock( return_value=True) # Don't try to find peers from the net def cleanup(): site_temp.delete() site_temp.content_manager.contents.db.close("Test cleanup") site_temp.content_manager.contents.db.timer_check_optional.kill() db_path = "%s-temp/content.db" % config.data_dir os.unlink(db_path) del ContentDb.content_dbs[db_path] gevent.killall([ obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before ]) request.addfinalizer(cleanup) site_temp.log = logging.getLogger("Temp:%s" % site_temp.address_short) return site_temp
def minikube(request, worker_id): def teardown(): if not k8s_skip_teardown: try: print("Removing minikube container ...") inst.container.remove(force=True, v=True) except: # noqa pylint: disable=bare-except pass request.addfinalizer(teardown) k8s_version = "v" + request.param.lstrip("v") k8s_timeout = int(request.config.getoption("--k8s-timeout")) k8s_container = request.config.getoption("--k8s-container") k8s_skip_teardown = request.config.getoption("--k8s-skip-teardown") inst = Minikube() inst.worker_id = worker_id if k8s_container: inst.connect(k8s_container, k8s_timeout) k8s_skip_teardown = True elif worker_id in ("master", "gw0"): inst.deploy(k8s_version, k8s_timeout) if worker_id == "gw0": k8s_skip_teardown = True else: inst.connect(K8S_DEFAULT_CONTAINER_NAME, k8s_timeout, version=k8s_version) k8s_skip_teardown = True return inst
def file_server4(request): time.sleep(0.1) file_server = FileServer("127.0.0.1", 1544) file_server.ip_external = "1.2.3.4" # Fake external ip def listen(): ConnectionServer.start(file_server) ConnectionServer.listen(file_server) gevent.spawn(listen) # Wait for port opening for retry in range(10): time.sleep(0.1) # Port opening try: conn = file_server.getConnection("127.0.0.1", 1544) conn.close() break except Exception as err: print("FileServer6 startup error", Debug.formatException(err)) assert file_server.running file_server.ip_incoming = {} # Reset flood protection def stop(): file_server.stop() request.addfinalizer(stop) return file_server
def before_tests(request): # Download files for local use urllib.request.urlretrieve("https://software.broadinstitute.org/cancer/software/genepattern/data/all_aml/all_aml_test.gct", "all_aml_test.gct") urllib.request.urlretrieve("https://software.broadinstitute.org/cancer/software/genepattern/data/protocols/all_aml_test.preprocessed.comp.marker.odf", "all_aml_test.preprocessed.comp.marker.odf") # Clean up after ourselves request.addfinalizer(after_tests)
def httpd(request): class RequestHandler(http.server.SimpleHTTPRequestHandler): def do_POST(self): logging.info('\n%s\n%s', self.requestline, self.headers) self.do_GET() def do_GET(self): logging.info('\n%s\n%s', self.requestline, self.headers) if self.path == '/site5/redirect/': self.send_response(303, 'See other') self.send_header('Connection', 'close') self.send_header('Content-Length', 0) self.send_header('Location', '/site5/destination/') self.end_headers() self.wfile.write(b'') elif self.path == '/site9/redirect.html': self.send_response(303, 'See other') self.send_header('Connection', 'close') self.send_header('Content-Length', 0) self.send_header('Location', '/site9/destination.html') self.end_headers() self.wfile.write(b'') elif self.path.startswith('/infinite/'): payload = b''' <html> <head> <title>infinite site</title> </head> <body> <a href='a/'>a/</a> <a href='b/'>b/</a> <a href='c/'>c/</a> <a href='d/'>d/</a> <a href='e/'>e/</a> <a href='f/'>f/</a> <a href='g/'>g/</a> <a href='h/'>h/</a> <a href='i/'>i/</a> </body> </html> ''' self.send_response(200, 'OK') self.send_header('Connection', 'close') self.send_header('Content-Length', len(payload)) self.end_headers() self.wfile.write(payload) else: super().do_GET() # SimpleHTTPRequestHandler always uses CWD so we have to chdir os.chdir(os.path.join(os.path.dirname(__file__), 'htdocs')) httpd = http.server.HTTPServer((local_address, 0), RequestHandler) httpd_thread = threading.Thread(name='httpd', target=httpd.serve_forever) httpd_thread.start() def fin(): httpd.shutdown() httpd.server_close() httpd_thread.join() request.addfinalizer(fin) return httpd
def client(request): """ Creates client fixture which can be used to access Flask API @returns: instance of testable flask app """ test_client = app.test_client() def teardown(): pass request.addfinalizer(teardown) return test_client
def teardown_test_data_dir(request): """ TEARDOWN: --------- 1) Remove the folder (+ all contents) at the end of the test suite. """ # Teardown def remove_test_dir_and_db(): shutil.rmtree(TEST_DATA_PATH) request.addfinalizer(remove_test_dir_and_db)
def httpd(request): class RequestHandler(http.server.SimpleHTTPRequestHandler): def do_GET(self): if self.path == '/site5/redirect/': self.send_response(303, 'See other') self.send_header('Connection', 'close') self.send_header('Content-Length', 0) self.send_header('Location', '/site5/destination/') self.end_headers() self.wfile.write(b'') elif self.path == '/site9/redirect.html': self.send_response(303, 'See other') self.send_header('Connection', 'close') self.send_header('Content-Length', 0) self.send_header('Location', '/site9/destination.html') self.end_headers() self.wfile.write(b'') elif self.path.startswith('/infinite/'): payload = b''' <html> <head> <title>infinite site</title> </head> <body> <a href='a/'>a/</a> <a href='b/'>b/</a> <a href='c/'>c/</a> <a href='d/'>d/</a> <a href='e/'>e/</a> <a href='f/'>f/</a> <a href='g/'>g/</a> <a href='h/'>h/</a> <a href='i/'>i/</a> </body> </html> ''' self.send_response(200, 'OK') self.send_header('Connection', 'close') self.send_header('Content-Length', len(payload)) self.end_headers() self.wfile.write(payload) else: super().do_GET() # SimpleHTTPRequestHandler always uses CWD so we have to chdir os.chdir(os.path.join(os.path.dirname(__file__), 'htdocs')) httpd = http.server.HTTPServer(('localhost', 0), RequestHandler) httpd_thread = threading.Thread(name='httpd', target=httpd.serve_forever) httpd_thread.start() def fin(): httpd.shutdown() httpd.server_close() httpd_thread.join() request.addfinalizer(fin) return httpd
async def principal(request, event_loop, caldav): """principal async fixture.""" principal = await caldav.principal() def finalize(): async def afin(): await principal.prune() event_loop.run_until_complete(afin()) request.addfinalizer(finalize) return principal
def wms_server(request): """ Run the WMS server for the duration of these tests """ external_url = os.environ.get("SERVER_URL") if external_url: server = generic_obj() server.url = external_url else: server = WSGIServer(application=wms.app) server.start() request.addfinalizer(server.stop) return server
def block_remote_conections(request): """ Block all remote calls during unit tests. """ # inspired by https://stackoverflow.com/q/18601828 def block_socket_side_effect(*args, **kwargs): raise ValueError(_BLOCKED_CONNECTION_MESSAGE) block_socket_patcher = mock.patch('socket.socket', side_effect=block_socket_side_effect) block_socket_patcher.start() request.addfinalizer(block_socket_patcher.stop)
def httpd(request): # SimpleHTTPRequestHandler always uses CWD so we have to chdir os.chdir(os.path.join(os.path.dirname(__file__), 'htdocs')) httpd = http.server.HTTPServer( ('localhost', 0), http.server.SimpleHTTPRequestHandler) httpd_thread = threading.Thread(name='httpd', target=httpd.serve_forever) httpd_thread.start() def fin(): httpd.shutdown() httpd.server_close() httpd_thread.join() request.addfinalizer(fin) return httpd
def browser(request): try: from selenium import webdriver print("Starting chromedriver...") options = webdriver.chrome.options.Options() options.add_argument("--headless") options.add_argument("--window-size=1920x1080") options.add_argument("--log-level=1") browser = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, service_log_path=os.path.devnull, options=options) def quit(): browser.quit() request.addfinalizer(quit) except Exception as err: raise pytest.skip("Test requires selenium + chromedriver: %s" % err) return browser
def httpd(request): # SimpleHTTPRequestHandler always uses CWD so we have to chdir os.chdir(os.path.join(os.path.dirname(__file__), 'htdocs')) httpd = http.server.HTTPServer( ('localhost', 0), http.server.SimpleHTTPRequestHandler) httpd_thread = threading.Thread(name='httpd', target=httpd.serve_forever) httpd_thread.start() def fin(): httpd.shutdown() httpd.server_close() httpd_thread.join() request.addfinalizer(fin) return httpd
def db(request): db_path = "%s/zeronet.db" % config.data_dir schema = { "db_name": "TestDb", "db_file": "%s/zeronet.db" % config.data_dir, "maps": { "data.json": { "to_table": [ "test", { "node": "test", "table": "test_importfilter", "import_cols": ["test_id", "title"] } ] } }, "tables": { "test": { "cols": [["test_id", "INTEGER"], ["title", "TEXT"], ["json_id", "INTEGER REFERENCES json (json_id)"]], "indexes": ["CREATE UNIQUE INDEX test_id ON test(test_id)"], "schema_changed": 1426195822 }, "test_importfilter": { "cols": [["test_id", "INTEGER"], ["title", "TEXT"], ["json_id", "INTEGER REFERENCES json (json_id)"]], "indexes": [ "CREATE UNIQUE INDEX test_importfilter_id ON test_importfilter(test_id)" ], "schema_changed": 1426195822 } } } if os.path.isfile(db_path): os.unlink(db_path) db = Db.Db(schema, db_path) db.checkTables() def stop(): db.close("Test db cleanup") os.unlink(db_path) request.addfinalizer(stop) return db
def agent_image(minikube, registry, request, worker_id): def teardown(): try: client.images.remove("%s:%s" % (agent_image_name, agent_image_tag)) except: pass request.addfinalizer(teardown) client = get_docker_client() final_agent_image_name = request.config.getoption("--k8s-agent-name") final_agent_image_tag = request.config.getoption("--k8s-agent-tag") agent_image_name = "localhost:%d/%s" % ( registry['port'], final_agent_image_name.split("/")[-1]) agent_image_tag = final_agent_image_tag if worker_id == "master" or worker_id == "gw0": if not has_docker_image(client, final_agent_image_name, final_agent_image_tag): print("\nAgent image '%s:%s' not found in local registry." % (final_agent_image_name, final_agent_image_tag)) print("\nAttempting to pull from remote registry ...") final_agent_image = client.images.pull(final_agent_image_name, tag=final_agent_image_tag) else: final_agent_image = client.images.get(final_agent_image_name + ":" + final_agent_image_tag) print("\nTagging %s:%s as %s:%s ..." % (final_agent_image_name, final_agent_image_tag, agent_image_name, agent_image_tag)) final_agent_image.tag(agent_image_name, tag=agent_image_tag) print("\nPushing %s:%s ..." % (agent_image_name, agent_image_tag)) client.images.push(agent_image_name, tag=agent_image_tag) print("\nPulling %s:%s ..." % (agent_image_name, agent_image_tag)) minikube.pull_agent_image(agent_image_name, agent_image_tag, final_agent_image.id) else: print( "\nWaiting for agent image \"%s:%s\" to be pulled to minikube ..." % (agent_image_name, agent_image_tag)) assert wait_for(p(has_docker_image, minikube.client, agent_image_name, agent_image_tag), timeout_seconds=60), \ "timed out waiting for agent image \"%s:%s\"!" % (agent_image_name, agent_image_tag) final_agent_image = client.images.get(final_agent_image_name + ":" + final_agent_image_tag) return { "name": agent_image_name, "tag": agent_image_tag, "id": final_agent_image.id }
def setup_index(request, index_name): es = Elasticsearch() try: es.indices.create(index=index_name) except RequestError as e: if e.error == u'index_already_exists_exception': es.indices.delete(index_name) else: raise def fin(): try: es.indices.delete(index_name) except NotFoundError: pass request.addfinalizer(fin)
def setup_index(request, index_name): es = Elasticsearch() try: es.indices.create(index=index_name) except RequestError as e: if e.error == u'index_already_exists_exception': es.indices.delete(index_name) else: raise def fin(): try: es.indices.delete(index_name) except NotFoundError: pass request.addfinalizer(fin)
def download_and_extract(request, req, target): cmd = [ sys.executable, '-m', 'pip', 'download', '--no-deps', '--no-binary', ':all:', req, ] output = subprocess.check_output(cmd, encoding='utf-8') filename = re.search('Saved (.*)', output).group(1) request.addfinalizer(functools.partial(os.remove, filename)) opener = zipfile.ZipFile if filename.endswith('.zip') else tarfile.open with opener(filename) as archive: archive.extractall(target) return os.path.join(target, os.listdir(target)[0])
def resetTempSettings(request): data_dir_temp = config.data_dir + "-temp" if not os.path.isdir(data_dir_temp): os.mkdir(data_dir_temp) open("%s/sites.json" % data_dir_temp, "w").write("{}") open("%s/filters.json" % data_dir_temp, "w").write("{}") open("%s/users.json" % data_dir_temp, "w").write(""" { "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": { "certs": {}, "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a", "sites": {} } } """) def cleanup(): os.unlink("%s/sites.json" % data_dir_temp) os.unlink("%s/users.json" % data_dir_temp) os.unlink("%s/filters.json" % data_dir_temp) request.addfinalizer(cleanup)
def site_temp(request): threads_before = [ obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) ] with mock.patch("Config.config.data_dir", config.data_dir + "-temp"): site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") site_temp.announce = mock.MagicMock( return_value=True) # Don't try to find peers from the net def cleanup(): site_temp.storage.deleteFiles() site_temp.content_manager.contents.db.deleteSite(site_temp) site_temp.content_manager.contents.db.close() time.sleep(0.01) # Wait for db close db_path = "%s-temp/content.db" % config.data_dir os.unlink(db_path) del ContentDb.content_dbs[db_path] gevent.killall([ obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before ]) request.addfinalizer(cleanup) return site_temp
def agent_image(minikube, registry, request, worker_id): # pylint: disable=redefined-outer-name def teardown(): if temp_agent_name and temp_agent_tag: try: client.images.remove("%s:%s" % (temp_agent_name, temp_agent_tag)) except: # noqa pylint: disable=bare-except pass request.addfinalizer(teardown) sfx_agent_name = request.config.getoption("--k8s-sfx-agent") if sfx_agent_name: try: agent_image_name, agent_image_tag = sfx_agent_name.rsplit( ":", maxsplit=1) except ValueError: agent_image_name = sfx_agent_name agent_image_tag = "latest" else: agent_image_name = "signalfx-agent" agent_image_tag = "k8s-test" temp_agent_name = None temp_agent_tag = None client = get_docker_client() if worker_id in ("master", "gw0"): if sfx_agent_name and not has_docker_image(client, sfx_agent_name): print('\nAgent image "%s" not found in local registry.' % sfx_agent_name) print("Attempting to pull from remote registry to minikube ...") sfx_agent_image = minikube.pull_agent_image( agent_image_name, agent_image_tag) _, output = minikube.container.exec_run("docker images") print(output.decode("utf-8")) return { "name": agent_image_name, "tag": agent_image_tag, "id": sfx_agent_image.id } if sfx_agent_name: print('\nAgent image "%s" found in local registry.' % sfx_agent_name) sfx_agent_image = client.images.get(sfx_agent_name) else: print( '\nBuilding agent image from local source and tagging as "%s:%s" ...' % (agent_image_name, agent_image_tag)) subprocess.run( "make image", shell=True, env={ "PULL_CACHE": "yes", "AGENT_IMAGE_NAME": agent_image_name, "AGENT_VERSION": agent_image_tag }, stderr=subprocess.STDOUT, check=True, ) sfx_agent_image = client.images.get(agent_image_name + ":" + agent_image_tag) temp_agent_name = "localhost:%d/signalfx-agent-dev" % registry["port"] temp_agent_tag = "latest" print("\nPushing agent image to minikube ...") sfx_agent_image.tag(temp_agent_name, tag=temp_agent_tag) client.images.push(temp_agent_name, tag=temp_agent_tag) sfx_agent_image = minikube.pull_agent_image(temp_agent_name, temp_agent_tag, sfx_agent_image.id) sfx_agent_image.tag(agent_image_name, tag=agent_image_tag) _, output = minikube.container.exec_run("docker images") print(output.decode("utf-8")) else: print("\nWaiting for agent image to be built/pulled to minikube ...") assert wait_for( p(has_docker_image, minikube.client, agent_image_name, agent_image_tag), timeout_seconds=600, interval_seconds=2, ), 'timed out waiting for agent image "%s:%s"!' % (agent_image_name, agent_image_tag) sfx_agent_image = minikube.client.images.get(agent_image_name + ":" + agent_image_tag) return { "name": agent_image_name, "tag": agent_image_tag, "id": sfx_agent_image.id }