def ask_input(s):
orig_handler = signal.getsignal(signal.SIGINT)
# allow Ctrl+C to throw KeyboardInterrupt in callback
signal.signal(signal.SIGINT, sigint_handler)
try:
return six_input(s)
finally:
signal.signal(signal.SIGINT, orig_handler)
def build_static_site(arguments, config):
"""
Do the actual work of building a static site
"""
# create Shortstack instance
location = config['location']
url_root = config['url_root']
destination = arguments.destination
application = Shortstack('shortstack',
instance_path=location,
url_root=url_root)
client = Client(application, BaseResponse)
print("Generating static site at: %s" % destination)
if os.path.exists(destination):
prompt = "Really delete (and rebuild) %s ? [y/n] " % destination
really_delete = six_input(prompt).lower() == 'y'
if really_delete:
shutil.rmtree(arguments.destination)
else:
return
for url in application.filtered_urls_from_filesystem():
save_document(client, arguments.destination, url)
def _create_postgres_test_db(cls):
""" Creates new database for testing. """
try:
pg_settings = json.load(open('.pg_settings.json'))
except IOError:
msg = 'Postgres database credentials was not found. Create `.pg_settings.json` file'\
' near setup.py. The minimal content of the .pg_settings.json:\n{}'.format(SETTINGS_EXAMPLE)
raise RuntimeError(msg)
# pg settings validation
missing = []
for field in ('user', 'password', 'host'):
if field not in pg_settings:
missing.append(field)
if missing:
msg = '.pg_settings.json is not valid. Add {} field(s). The minimal example: \n{}'\
.format(', '.join(missing), SETTINGS_EXAMPLE)
raise RuntimeError(msg)
user = pg_settings['user']
password = pg_settings['password']
host = pg_settings['host']
test_db_name = 'ambry_test_{}'.format(SAFETY_POSTFIX)
postgres_db_name = 'postgres'
# connect to postgres database because we need to create database for tests.
with psycopg2.connect(host=host, database=postgres_db_name, user=user, password=password) as conn:
with conn.cursor() as curs:
# we have to close opened transaction.
curs.execute('COMMIT;')
# drop test database created by previuos run (control + c case).
if cls.postgres_db_exists(test_db_name, curs):
assert test_db_name.endswith(SAFETY_POSTFIX), 'Can not drop database without safety postfix.'
while True:
delete_it = six_input(
'\nTest database with {} name already exists. Can I delete it (Yes|No): '.format(test_db_name))
if delete_it.lower() == 'yes':
try:
curs.execute('DROP DATABASE {};'.format(test_db_name))
curs.execute('COMMIT;')
except:
curs.execute('ROLLBACK;')
break
elif delete_it.lower() == 'no':
break
#
# check for test template with required extensions.
TEMPLATE_NAME = 'template0_ambry_test'
cls.test_template_exists = cls.postgres_db_exists(TEMPLATE_NAME, curs)
if not cls.test_template_exists:
raise unittest.SkipTest(
'Tests require custom postgres template db named {}. '
'See DEVEL-README.md for details.'.format(TEMPLATE_NAME))
query = 'CREATE DATABASE {} OWNER {} TEMPLATE {} encoding \'UTF8\';'\
.format(test_db_name, user, TEMPLATE_NAME)
curs.execute(query)
curs.execute('COMMIT;')
# reconnect to test db and create schemas needed by ambry_sources.
with psycopg2.connect(host=host, database=test_db_name, user=user, password=password) as conn:
with conn.cursor() as curs:
# we have to close opened transaction.
curs.execute('COMMIT;')
curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(POSTGRES_SCHEMA_NAME))
curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(POSTGRES_PARTITION_SCHEMA_NAME))
if not cls.postgres_extension_installed('pg_trgm', curs):
raise unittest.SkipTest(
'Can not find template with pg_trgm extension. See DEVEL-README.md for details.')
if not cls.postgres_extension_installed('multicorn', curs):
raise unittest.SkipTest(
'Can not find template with multicorn extension. See DEVEL-README.md for details.')
cls.pg_postgres_db_data = {
'database': postgres_db_name,
'user': user,
'host': host,
'password': password
}
cls.pg_test_db_data = {
'database': test_db_name,
'user': user,
'host': host,
'password': password
}
return (cls.pg_postgres_db_data, cls.pg_test_db_data)
def main():
opts = parse_args()
if not os.path.exists(opts.destination_path):
msg = ('Directory "{}" does not exist; should I create it? [y/N] '
''.format(opts.destination_path))
create = six_input(msg).lower().strip() == 'y'
if create:
os.makedirs(opts.destination_path)
else:
print('Aborting.')
exit(1)
if len(os.listdir(opts.destination_path)) > 0:
msg = ('Directory "{}" is not empty; should I empty it? [y/N] '
''.format(opts.destination_path))
empty = six_input(msg).lower().strip() == 'y'
if empty:
shutil.rmtree(opts.destination_path)
os.mkdir(opts.destination_path)
else:
print('Aborting.')
exit(1)
if opts.normalize_unicode:
try:
unidecode
except NameError:
err = ('`unidecode` is needed for unicode normalization'
'please install it via the `[sudo] pip install '
'unidecode` command.')
print(err, file=sys.stderr)
exit(1)
flag_fp = os.path.join(opts.destination_path, 'normalize-unicode.flag')
open(flag_fp, 'w').close()
if opts.lowercase:
flag_fp = os.path.join(opts.destination_path, 'lowercase.flag')
open(flag_fp, 'w').close()
flag_fp = os.path.join(opts.destination_path, 'language.flag')
with open(flag_fp, 'w') as f:
f.write(opts.language)
mrconso_path = os.path.join(opts.umls_installation_path, 'MRCONSO.RRF')
mrsty_path = os.path.join(opts.umls_installation_path, 'MRSTY.RRF')
mrconso_iterator = extract_from_mrconso(mrconso_path, mrsty_path, opts)
simstring_dir = os.path.join(opts.destination_path, 'umls-simstring.db')
cuisty_dir = os.path.join(opts.destination_path, 'cui-semtypes.db')
cuipt_dir = None
if opts.save_preferred_terms is True:
cuipt_dir = os.path.join(opts.destination_path,
'cui-preferred-terms.db')
parse_and_encode_ngrams(mrconso_iterator,
simstring_dir,
cuisty_dir,
cuipt_dir=cuipt_dir)
def _create_postgres_test_db(cls):
""" Creates new database for testing. """
try:
pg_settings = json.load(open('.pg_settings.json'))
except IOError:
msg = 'Postgres database credentials was not found. Create `.pg_settings.json` file'\
' near setup.py. The minimal content of the .pg_settings.json:\n{}'.format(SETTINGS_EXAMPLE)
raise RuntimeError(msg)
# pg settings validation
missing = []
for field in ('user', 'password', 'host'):
if field not in pg_settings:
missing.append(field)
if missing:
msg = '.pg_settings.json is not valid. Add {} field(s). The minimal example: \n{}'\
.format(', '.join(missing), SETTINGS_EXAMPLE)
raise RuntimeError(msg)
user = pg_settings['user']
password = pg_settings['password']
host = pg_settings['host']
test_db_name = 'ambry_test_{}'.format(SAFETY_POSTFIX)
postgres_db_name = 'postgres'
# connect to postgres database because we need to create database for tests.
with psycopg2.connect(host=host,
database=postgres_db_name,
user=user,
password=password) as conn:
with conn.cursor() as curs:
# we have to close opened transaction.
curs.execute('COMMIT;')
# drop test database created by previuos run (control + c case).
if cls.postgres_db_exists(test_db_name, curs):
assert test_db_name.endswith(
SAFETY_POSTFIX
), 'Can not drop database without safety postfix.'
while True:
delete_it = six_input(
'\nTest database with {} name already exists. Can I delete it (Yes|No): '
.format(test_db_name))
if delete_it.lower() == 'yes':
try:
curs.execute(
'DROP DATABASE {};'.format(test_db_name))
curs.execute('COMMIT;')
except:
curs.execute('ROLLBACK;')
break
elif delete_it.lower() == 'no':
break
#
# check for test template with required extensions.
TEMPLATE_NAME = 'template0_ambry_test'
cls.test_template_exists = cls.postgres_db_exists(
TEMPLATE_NAME, curs)
if not cls.test_template_exists:
raise unittest.SkipTest(
'Tests require custom postgres template db named {}. '
'See DEVEL-README.md for details.'.format(
TEMPLATE_NAME))
query = 'CREATE DATABASE {} OWNER {} TEMPLATE {} encoding \'UTF8\';'\
.format(test_db_name, user, TEMPLATE_NAME)
curs.execute(query)
curs.execute('COMMIT;')
# reconnect to test db and create schemas needed by ambry_sources.
with psycopg2.connect(host=host,
database=test_db_name,
user=user,
password=password) as conn:
with conn.cursor() as curs:
# we have to close opened transaction.
curs.execute('COMMIT;')
curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(
POSTGRES_SCHEMA_NAME))
curs.execute('CREATE SCHEMA IF NOT EXISTS {};'.format(
POSTGRES_PARTITION_SCHEMA_NAME))
if not cls.postgres_extension_installed('pg_trgm', curs):
raise unittest.SkipTest(
'Can not find template with pg_trgm extension. See DEVEL-README.md for details.'
)
if not cls.postgres_extension_installed('multicorn', curs):
raise unittest.SkipTest(
'Can not find template with multicorn extension. See DEVEL-README.md for details.'
)
cls.pg_postgres_db_data = {
'database': postgres_db_name,
'user': user,
'host': host,
'password': password
}
cls.pg_test_db_data = {
'database': test_db_name,
'user': user,
'host': host,
'password': password
}
return (cls.pg_postgres_db_data, cls.pg_test_db_data)
def ask_input(s):
return six_input(s)