def ask_input(s): orig_handler = signal.getsignal(signal.SIGINT) # allow Ctrl+C to throw KeyboardInterrupt in callback signal.signal(signal.SIGINT, sigint_handler) try: return six_input(s) finally: signal.signal(signal.SIGINT, orig_handler)
def build_static_site(arguments, config): """ Do the actual work of building a static site """ # create Shortstack instance location = config['location'] url_root = config['url_root'] destination = arguments.destination application = Shortstack('shortstack', instance_path=location, url_root=url_root) client = Client(application, BaseResponse) print("Generating static site at: %s" % destination) if os.path.exists(destination): prompt = "Really delete (and rebuild) %s ? [y/n] " % destination really_delete = six_input(prompt).lower() == 'y' if really_delete: shutil.rmtree(arguments.destination) else: return for url in application.filtered_urls_from_filesystem(): save_document(client, arguments.destination, url)
def _create_postgres_test_db(cls): """ Creates new database for testing. """ try: pg_settings = json.load(open('.pg_settings.json')) except IOError: msg = 'Postgres database credentials was not found. Create `.pg_settings.json` file'\ ' near setup.py. The minimal content of the .pg_settings.json:\n{}'.format(SETTINGS_EXAMPLE) raise RuntimeError(msg) # pg settings validation missing = [] for field in ('user', 'password', 'host'): if field not in pg_settings: missing.append(field) if missing: msg = '.pg_settings.json is not valid. Add {} field(s). The minimal example: \n{}'\ .format(', '.join(missing), SETTINGS_EXAMPLE) raise RuntimeError(msg) user = pg_settings['user'] password = pg_settings['password'] host = pg_settings['host'] test_db_name = 'ambry_test_{}'.format(SAFETY_POSTFIX) postgres_db_name = 'postgres' # connect to postgres database because we need to create database for tests. with psycopg2.connect(host=host, database=postgres_db_name, user=user, password=password) as conn: with conn.cursor() as curs: # we have to close opened transaction. curs.execute('COMMIT;') # drop test database created by previuos run (control + c case). if cls.postgres_db_exists(test_db_name, curs): assert test_db_name.endswith(SAFETY_POSTFIX), 'Can not drop database without safety postfix.' while True: delete_it = six_input( '\nTest database with {} name already exists. Can I delete it (Yes|No): '.format(test_db_name)) if delete_it.lower() == 'yes': try: curs.execute('DROP DATABASE {};'.format(test_db_name)) curs.execute('COMMIT;') except: curs.execute('ROLLBACK;') break elif delete_it.lower() == 'no': break # # check for test template with required extensions. TEMPLATE_NAME = 'template0_ambry_test' cls.test_template_exists = cls.postgres_db_exists(TEMPLATE_NAME, curs) if not cls.test_template_exists: raise unittest.SkipTest( 'Tests require custom postgres template db named {}. ' 'See DEVEL-README.md for details.'.format(TEMPLATE_NAME)) query = 'CREATE DATABASE {} OWNER {} TEMPLATE {} encoding \'UTF8\';'\ .format(test_db_name, user, TEMPLATE_NAME) curs.execute(query) curs.execute('COMMIT;') # reconnect to test db and create schemas needed by ambry_sources. with psycopg2.connect(host=host, database=test_db_name, user=user, password=password) as conn: with conn.cursor() as curs: # we have to close opened transaction. curs.execute('COMMIT;') curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(POSTGRES_SCHEMA_NAME)) curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(POSTGRES_PARTITION_SCHEMA_NAME)) if not cls.postgres_extension_installed('pg_trgm', curs): raise unittest.SkipTest( 'Can not find template with pg_trgm extension. See DEVEL-README.md for details.') if not cls.postgres_extension_installed('multicorn', curs): raise unittest.SkipTest( 'Can not find template with multicorn extension. See DEVEL-README.md for details.') cls.pg_postgres_db_data = { 'database': postgres_db_name, 'user': user, 'host': host, 'password': password } cls.pg_test_db_data = { 'database': test_db_name, 'user': user, 'host': host, 'password': password } return (cls.pg_postgres_db_data, cls.pg_test_db_data)
def main(): opts = parse_args() if not os.path.exists(opts.destination_path): msg = ('Directory "{}" does not exist; should I create it? [y/N] ' ''.format(opts.destination_path)) create = six_input(msg).lower().strip() == 'y' if create: os.makedirs(opts.destination_path) else: print('Aborting.') exit(1) if len(os.listdir(opts.destination_path)) > 0: msg = ('Directory "{}" is not empty; should I empty it? [y/N] ' ''.format(opts.destination_path)) empty = six_input(msg).lower().strip() == 'y' if empty: shutil.rmtree(opts.destination_path) os.mkdir(opts.destination_path) else: print('Aborting.') exit(1) if opts.normalize_unicode: try: unidecode except NameError: err = ('`unidecode` is needed for unicode normalization' 'please install it via the `[sudo] pip install ' 'unidecode` command.') print(err, file=sys.stderr) exit(1) flag_fp = os.path.join(opts.destination_path, 'normalize-unicode.flag') open(flag_fp, 'w').close() if opts.lowercase: flag_fp = os.path.join(opts.destination_path, 'lowercase.flag') open(flag_fp, 'w').close() flag_fp = os.path.join(opts.destination_path, 'language.flag') with open(flag_fp, 'w') as f: f.write(opts.language) mrconso_path = os.path.join(opts.umls_installation_path, 'MRCONSO.RRF') mrsty_path = os.path.join(opts.umls_installation_path, 'MRSTY.RRF') mrconso_iterator = extract_from_mrconso(mrconso_path, mrsty_path, opts) simstring_dir = os.path.join(opts.destination_path, 'umls-simstring.db') cuisty_dir = os.path.join(opts.destination_path, 'cui-semtypes.db') cuipt_dir = None if opts.save_preferred_terms is True: cuipt_dir = os.path.join(opts.destination_path, 'cui-preferred-terms.db') parse_and_encode_ngrams(mrconso_iterator, simstring_dir, cuisty_dir, cuipt_dir=cuipt_dir)
def _create_postgres_test_db(cls): """ Creates new database for testing. """ try: pg_settings = json.load(open('.pg_settings.json')) except IOError: msg = 'Postgres database credentials was not found. Create `.pg_settings.json` file'\ ' near setup.py. The minimal content of the .pg_settings.json:\n{}'.format(SETTINGS_EXAMPLE) raise RuntimeError(msg) # pg settings validation missing = [] for field in ('user', 'password', 'host'): if field not in pg_settings: missing.append(field) if missing: msg = '.pg_settings.json is not valid. Add {} field(s). The minimal example: \n{}'\ .format(', '.join(missing), SETTINGS_EXAMPLE) raise RuntimeError(msg) user = pg_settings['user'] password = pg_settings['password'] host = pg_settings['host'] test_db_name = 'ambry_test_{}'.format(SAFETY_POSTFIX) postgres_db_name = 'postgres' # connect to postgres database because we need to create database for tests. with psycopg2.connect(host=host, database=postgres_db_name, user=user, password=password) as conn: with conn.cursor() as curs: # we have to close opened transaction. curs.execute('COMMIT;') # drop test database created by previuos run (control + c case). if cls.postgres_db_exists(test_db_name, curs): assert test_db_name.endswith( SAFETY_POSTFIX ), 'Can not drop database without safety postfix.' while True: delete_it = six_input( '\nTest database with {} name already exists. Can I delete it (Yes|No): ' .format(test_db_name)) if delete_it.lower() == 'yes': try: curs.execute( 'DROP DATABASE {};'.format(test_db_name)) curs.execute('COMMIT;') except: curs.execute('ROLLBACK;') break elif delete_it.lower() == 'no': break # # check for test template with required extensions. TEMPLATE_NAME = 'template0_ambry_test' cls.test_template_exists = cls.postgres_db_exists( TEMPLATE_NAME, curs) if not cls.test_template_exists: raise unittest.SkipTest( 'Tests require custom postgres template db named {}. ' 'See DEVEL-README.md for details.'.format( TEMPLATE_NAME)) query = 'CREATE DATABASE {} OWNER {} TEMPLATE {} encoding \'UTF8\';'\ .format(test_db_name, user, TEMPLATE_NAME) curs.execute(query) curs.execute('COMMIT;') # reconnect to test db and create schemas needed by ambry_sources. with psycopg2.connect(host=host, database=test_db_name, user=user, password=password) as conn: with conn.cursor() as curs: # we have to close opened transaction. curs.execute('COMMIT;') curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format( POSTGRES_SCHEMA_NAME)) curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format( POSTGRES_PARTITION_SCHEMA_NAME)) if not cls.postgres_extension_installed('pg_trgm', curs): raise unittest.SkipTest( 'Can not find template with pg_trgm extension. See DEVEL-README.md for details.' ) if not cls.postgres_extension_installed('multicorn', curs): raise unittest.SkipTest( 'Can not find template with multicorn extension. See DEVEL-README.md for details.' ) cls.pg_postgres_db_data = { 'database': postgres_db_name, 'user': user, 'host': host, 'password': password } cls.pg_test_db_data = { 'database': test_db_name, 'user': user, 'host': host, 'password': password } return (cls.pg_postgres_db_data, cls.pg_test_db_data)
def ask_input(s): return six_input(s)