def test_indexing_workbook(testapp, indexer_testapp): # First post a single item so that subsequent indexing is incremental testapp.post_json('/testing-post-put-patch/', {'required': ''}) res = indexer_testapp.post_json('/index', {'record': True}) assert res.json['indexed'] == 1 from encoded.loadxl import load_all from pkg_resources import resource_filename inserts = resource_filename('encoded', 'tests/data/inserts/') docsdir = [resource_filename('encoded', 'tests/data/documents/')] load_all(testapp, inserts, docsdir) res = indexer_testapp.post_json('/index', { 'record': True, 'is_testing_full': True }) assert res.json['updated'] assert res.json['indexed'] ### OPTIONAL: audit via 2-pass is coming... #assert res.json['pass2_took'] ### OPTIONAL: audit via 2-pass is coming... # NOTE: Both vis and region indexers are "followup" or secondary indexers # and must be staged by the primary indexer res = indexer_testapp.post_json('/index_vis', {'record': True}) assert res.json['cycle_took'] assert res.json['title'] == 'vis_indexer' res = indexer_testapp.post_json('/index_region', {'record': True}) assert res.json['cycle_took'] assert res.json['title'] == 'region_indexer' assert res.json['indexed'] > 0 res = testapp.get('/search/?type=Biosample') assert res.json['total'] > 5
def index_workbook(request, app): from snovault import DBSESSION connection = app.registry[DBSESSION].bind.pool.unique_connection() connection.detach() conn = connection.connection conn.autocommit = True cursor = conn.cursor() cursor.execute("""TRUNCATE resources, transactions CASCADE;""") cursor.close() from webtest import TestApp log_level = request.config.getoption("--log") environ = { 'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': '******', } testapp = TestApp(app, environ) from encoded.loadxl import load_all from pkg_resources import resource_filename inserts = resource_filename('encoded', 'tests/data/inserts/') docsdir = [resource_filename('encoded', 'tests/data/documents/')] load_all(testapp, inserts, docsdir, log_level=log_level) testapp.post_json('/index', {'is_testing_full': True}) yield
def workbook(conn, app, app_settings): tx = conn.begin_nested() try: from webtest import TestApp environ = { 'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': '******', } testapp = TestApp(app, environ) from encoded.loadxl import load_all from pkg_resources import resource_filename inserts = resource_filename('encoded', 'tests/data/inserts/') docsdir = [resource_filename('encoded', 'tests/data/documents/')] load_all(testapp, inserts, docsdir) yield finally: tx.rollback()
def main(): import argparse parser = argparse.ArgumentParser( description="Run development servers", epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('--app-name', help="Pyramid app name in configfile") parser.add_argument('config_uri', help="path to configfile") parser.add_argument('--clear', action="store_true", help="Clear existing data") parser.add_argument('--init', action="store_true", help="Init database") parser.add_argument('--load', action="store_true", help="Load test set") parser.add_argument('--datadir', default='/tmp/encoded', help="path to datadir") args = parser.parse_args() logging.basicConfig() # Loading app will have configured from config file. Reconfigure here: logging.getLogger('encoded').setLevel(logging.DEBUG) from encoded.tests import elasticsearch_fixture, postgresql_fixture from contentbase.elasticsearch import create_mapping datadir = os.path.abspath(args.datadir) pgdata = os.path.join(datadir, 'pgdata') esdata = os.path.join(datadir, 'esdata') if args.clear: for dirname in [pgdata, esdata]: if os.path.exists(dirname): shutil.rmtree(dirname) if args.init: postgresql_fixture.initdb(pgdata, echo=True) postgres = postgresql_fixture.server_process(pgdata, echo=True) elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True) processes = [postgres, elasticsearch] @atexit.register def cleanup_process(): for process in processes: if process.poll() is None: process.terminate() for process in processes: try: for line in process.stdout: sys.stdout.write(line.decode('utf-8')) except IOError: pass process.wait() if args.init: app = get_app(args.config_uri, args.app_name) create_mapping.run(app) if args.load: from webtest import TestApp environ = { 'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': '******', } testapp = TestApp(app, environ) from encoded.loadxl import load_all from pkg_resources import resource_filename inserts = resource_filename('encoded', 'tests/data/inserts/') docsdir = [resource_filename('encoded', 'tests/data/documents/')] load_all(testapp, inserts, docsdir) print('Started. ^C to exit.') stdouts = [p.stdout for p in processes] # Ugly should probably use threads instead while True: readable, writable, err = select.select(stdouts, [], stdouts, 5) for stdout in readable: for line in iter(stdout.readline, b''): sys.stdout.write(line.decode('utf-8')) if err: for stdout in err: for line in iter(stdout.readline, b''): sys.stdout.write(line.decode('utf-8')) break
def main(): import argparse parser = argparse.ArgumentParser( description="Run development servers", epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('--app-name', help="Pyramid app name in configfile") parser.add_argument('config_uri', help="path to configfile") parser.add_argument('--clear', action="store_true", help="Clear existing data") parser.add_argument('--init', action="store_true", help="Init database") parser.add_argument('--load', action="store_true", help="Load test set") parser.add_argument('--datadir', default='/tmp/encoded', help="path to datadir") args = parser.parse_args() logging.basicConfig() # Loading app will have configured from config file. Reconfigure here: logging.getLogger('encoded').setLevel(logging.DEBUG) from encoded.tests import elasticsearch_fixture, postgresql_fixture from encoded.commands import create_mapping datadir = os.path.abspath(args.datadir) pgdata = os.path.join(datadir, 'pgdata') esdata = os.path.join(datadir, 'esdata') if args.clear: for dirname in [pgdata, esdata]: if os.path.exists(dirname): shutil.rmtree(dirname) if args.init: postgresql_fixture.initdb(pgdata, echo=True) postgres = postgresql_fixture.server_process(pgdata, echo=True) elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True) processes = [postgres, elasticsearch] @atexit.register def cleanup_process(): for process in processes: if process.poll() is None: process.terminate() for process in processes: try: for line in process.stdout: sys.stdout.write(line) except IOError: pass process.wait() if args.init: app = get_app(args.config_uri, args.app_name) create_mapping.run(app) if args.load: from webtest import TestApp environ = { 'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': '******', } testapp = TestApp(app, environ) from encoded.loadxl import load_all from pkg_resources import resource_filename inserts = resource_filename('encoded', 'tests/data/inserts/') docsdir = [resource_filename('encoded', 'tests/data/documents/')] load_all(testapp, inserts, docsdir) print('Started. ^C to exit.') stdouts = [p.stdout for p in processes] # Ugly should probably use threads instead while True: readable, writable, err = select.select(stdouts, [], stdouts, 5) for stdout in readable: for line in iter(stdout.readline, ''): sys.stdout.write(line) if err: for stdout in err: for line in iter(stdout.readline, ''): sys.stdout.write(line) break
def main(): # https://github.com/gawel/WSGIProxy2/pull/3 (and change to WebTest) from wsgiproxy.proxies import ALLOWED_METHODS if 'PATCH' not in ALLOWED_METHODS: ALLOWED_METHODS.append('PATCH') import argparse parser = argparse.ArgumentParser( description="Import data", epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('--test-only', action='store_true') parser.add_argument('--item-type', help="Item type") parser.add_argument('--post', dest='method', action='store_const', const="POST", help="Create new data") parser.add_argument('--put', dest='method', action='store_const', const="PUT", help="Replace existing data") parser.add_argument('--patch', dest='method', action='store_const', const="PATCH", help="Patch existing data") parser.add_argument( '--username', '-u', default='', help="HTTP username (access_key_id) or import user uuid/email") parser.add_argument('--password', '-p', default='', help="HTTP password (secret_access_key)") parser.add_argument('--attach', '-a', action='append', default=[], help="Directory to search for attachments") parser.add_argument('--app-name', help="Pyramid app name in configfile") parser.add_argument( 'inpath', help="input zip file/directory of excel/csv/tsv sheets.") parser.add_argument( 'url', help="either the url to the application or path to configfile") args = parser.parse_args() logging.basicConfig() url = urlparse(args.url) if urlparse(args.url).scheme in ('http', 'https'): base = url.scheme + '://' + url.netloc username = args.username password = args.password if url.username: base = url.scheme + '://' + url.netloc.split('@', 1)[1] assert not args.username username = url.username if url.password: assert not args.password password = url.password testapp = remote_app(base, username, password) else: testapp = internal_app(args.url, args.app_name, args.username) # Loading app will have configured from config file. Reconfigure here: logging.getLogger('encoded').setLevel(logging.INFO) logging.getLogger('wsgi').setLevel(logging.WARNING) if args.method: run(testapp, args.inpath, args.attach, args.method, args.item_type, args.test_only) else: loadxl.load_all(testapp, args.inpath, args.attach, args.test_only)