def test_script_lower_than_current_version(self):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, '0001 0005'.split())
self.assertEqual(count(conn, 'test.hazard_calculation'), 2)
self.assertEqual(count(conn, 'test.lt_source_model'), 6)
# we support the use case of people reserving a number for a
# a script which is not ready yet: so, if I tell people to start
# numbering from 5 because I am working on the script #4,
# when I finally merge my upgrade will not be lost even if
# higher number scripts entered before it. One can even reserve
# a bunch of numbers, say from 0020 to 0029, for upgrades belonging
# to the same project.
# NB: if the higher number scripts contain incompatible changes
# the migration will fail; then you must fix the script and
# give it a higher number
# here we emulate this use case with a reserved script 04 entering
# when the database is already at version 05
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, ['0004'])
# lower version scripts are rejected by what_if_I_upgrade
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
with self.assertRaises(VersionTooSmall):
what_if_I_upgrade(conn, pkg)
def test_no_upgrades(self):
with self.assertRaises(SystemExit) as ctx:
upgrade_db(conn, 'openquake.engine.tests.db.no_upgrades')
self.assertTrue(
str(ctx.exception).startswith(
'The upgrade_dir does not contain scripts matching the pattern'
))
def test_insert_error(self):
with self.assertRaises(psycopg2.DataError):
# run 03-insert-error.sql
upgrade_db(conn, pkg, skip_versions=['0002'])
# check that the rollback works: the version
# table contains only the base script and the
# tables are not populated, i.e. '0001' was rolled back
self.assertEqual(count(conn, 'test.version'), 1)
self.assertEqual(count(conn, 'test.hazard_calculation'), 0)
self.assertEqual(count(conn, 'test.lt_source_model'), 0)
def test_insert_error(self):
with self.assertRaises(psycopg2.DataError):
# run 03-insert-error.sql
upgrade_db(conn, pkg, skip_versions=["0002"])
# check that the rollback works: the version
# table contains only the base script and the
# tables are not populated, i.e. '0001' was rolled back
self.assertEqual(count(conn, "test.version"), 1)
self.assertEqual(count(conn, "test.hazard_calculation"), 0)
self.assertEqual(count(conn, "test.lt_source_model"), 0)
def test_script_lower_than_current_version(self):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, '0001 0005'.split())
self.assertEqual(count(conn, 'test.hazard_calculation'), 2)
self.assertEqual(count(conn, 'test.lt_source_model'), 6)
# a script 0004 can enter when the database is already at version 0005
# (this is a convenient feature during development) but officially
# this is not supported and what_if_I_upgrade must raise an exception
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, ['0004'])
# check that the script 0004 is rejected by what_if_I_upgrade
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
with self.assertRaises(VersionTooSmall):
what_if_I_upgrade(conn, pkg, 'read_scripts')
def test_script_lower_than_current_version(self):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, '0001 0005'.split())
self.assertEqual(count(conn, 'test.hazard_calculation'), 2)
self.assertEqual(count(conn, 'test.lt_source_model'), 6)
# a script 0004 can enter when the database is already at version 0005
# (this is a convenient feature during development) but officially
# this is not supported and what_if_I_upgrade must raise an exception
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
applied = upgrade_db(conn, pkg, skip_versions='0002 0003'.split())
self.assertEqual(applied, ['0004'])
# check that the script 0004 is rejected by what_if_I_upgrade
with temp_script('0004-do-nothing.sql', 'SELECT 1'):
with self.assertRaises(VersionTooSmall):
what_if_I_upgrade(conn, pkg, 'read_scripts')
def test_ok(self):
applied = upgrade_db(conn, pkg, skip_versions="0002 0003".split())
self.assertEqual(applied, "0001 0005".split())
self.assertEqual(count(conn, "test.hazard_calculation"), 2)
self.assertEqual(count(conn, "test.lt_source_model"), 6)
# we support the use case of people reserving a number for a
# a script which is not ready yet: so, if I tell people to start
# numbering from 5 because I am working on the script #4,
# when I finally merge my upgrade will not be lost even if
# higher number scripts entered before it. One can even reserve
# a bunch of numbers, say from 0020 to 0029, for upgrades affering
# to the same project.
# NB: if the higher number scripts contain incompatible changes
# the migration will fail; then you must fix the script and
# give it a higher number
# here we emulate this use case with a reserved script 04 entering
# when the database is already at version 05
with temp_script("0004-do-nothing.sql", "SELECT 1"):
applied = upgrade_db(conn, pkg, skip_versions="0002 0003".split())
self.assertEqual(applied, ["0004"])
def test_duplicated_version(self):
# there are two scripts with version '0001'
with self.assertRaises(DuplicatedVersion):
with temp_script("0001-do-nothing.sql", "SELECT 1"):
upgrade_db(conn, pkg)
def main():
arg_parser = set_up_arg_parser()
args = arg_parser.parse_args()
exports = args.exports or 'xml,csv'
if args.version:
print __version__
sys.exit(0)
if args.run or args.run_hazard or args.run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if args.config_file:
os.environ[config.OQ_CONFIG_FILE_VAR] = \
abspath(expanduser(args.config_file))
config.refresh()
if args.no_distribute:
os.environ[openquake.engine.NO_DISTRIBUTE_VAR] = '1'
if args.make_html_report:
conn = models.getcursor('admin').connection
print 'Written', make_report(conn, args.make_html_report)
sys.exit(0)
if args.upgrade_db:
logs.set_level('info')
conn = models.getcursor('admin').connection
msg = upgrade_manager.what_if_I_upgrade(
conn, extract_scripts='read_scripts')
print msg
if msg.startswith('Your database is already updated'):
pass
elif args.yes or confirm('Proceed? (y/n) '):
upgrade_manager.upgrade_db(conn)
sys.exit(0)
if args.version_db:
conn = models.getcursor('admin').connection
print upgrade_manager.version_db(conn)
sys.exit(0)
if args.what_if_I_upgrade:
conn = models.getcursor('admin').connection
print upgrade_manager.what_if_I_upgrade(conn)
sys.exit(0)
run_job = engine.run_job_lite if args.lite else engine.run_job
if args.lite and args.hazard_output_id:
sys.exit('The --hazard-output-id option is not supported with the '
'--lite option')
if args.list_inputs:
list_inputs(args.list_inputs)
# hazard or hazard+risk
hc_id = args.hazard_calculation_id
if hc_id and int(hc_id) < 0:
# make it possible commands like `oq-engine --run job_risk.ini --hc -1`
hc_id = get_hc_id(int(hc_id))
if args.run:
job_inis = map(expanduser, args.run.split(','))
if len(job_inis) not in (1, 2):
sys.exit('%s should be a .ini filename or a pair of filenames '
'separated by a comma' % args.run)
for job_ini in job_inis:
open(job_ini).read() # raise an IOError if the file does not exist
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
if len(job_inis) == 2:
# run hazard
job = run_job(job_inis[0], args.log_level,
log_file, args.exports)
# run risk
run_job(job_inis[1], args.log_level, log_file,
args.exports, hazard_calculation_id=job.id)
else:
run_job(
expanduser(args.run), args.log_level, log_file,
args.exports, hazard_output_id=args.hazard_output_id,
hazard_calculation_id=hc_id)
# hazard
elif args.list_hazard_calculations:
list_calculations('hazard')
elif args.run_hazard is not None:
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
run_job(expanduser(args.run_hazard), args.log_level,
log_file, args.exports)
elif args.delete_hazard_calculation is not None:
del_calc(args.delete_hazard_calculation, args.yes)
# risk
elif args.list_risk_calculations:
list_calculations('risk')
elif args.run_risk is not None:
if (args.hazard_output_id is None and
args.hazard_calculation_id is None):
sys.exit(MISSING_HAZARD_MSG)
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
run_job(
expanduser(args.run_risk),
args.log_level, log_file, args.exports,
hazard_output_id=args.hazard_output_id,
hazard_calculation_id=hc_id)
elif args.delete_risk_calculation is not None:
del_calc(args.delete_risk_calculation, args.yes)
# export
elif args.list_outputs is not None:
engine.list_outputs(get_hc_id(args.list_outputs))
elif args.show_view is not None:
job_id, view_name = args.show_view
print views.view(view_name, get_hc_id(job_id))
elif args.list_hazard_outputs is not None:
deprecate('--list-hazard-outputs', '--list-outputs')
engine.list_outputs(args.list_hazard_outputs)
elif args.list_risk_outputs is not None:
deprecate('--list-risk-outputs', '--list-outputs')
engine.list_outputs(args.list_risk_outputs)
elif args.export_output is not None:
output_id, target_dir = args.export_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_hazard_output is not None:
deprecate('--export-hazard-output', '--export-output')
output_id, target_dir = args.export_hazard_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_risk_output is not None:
deprecate('--export-risk-output', '--export-output')
output_id, target_dir = args.export_risk_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_outputs is not None:
job_id, target_dir = args.export_outputs
export_outputs(get_hc_id(job_id), expanduser(target_dir), exports)
elif args.export_stats is not None:
job_id, target_dir, output_type = args.export_stats
export_stats(get_hc_id(job_id), expanduser(target_dir),
output_type, exports)
# deprecated
elif args.export_hazard_outputs is not None:
deprecate('--export-hazard-outputs', '--export-outputs')
job_id, target_dir = args.export_hazard_outputs
export_outputs(get_hc_id(job_id), expanduser(target_dir), exports)
elif args.export_risk_outputs is not None:
deprecate('--export-risk-outputs', '--export-outputs')
job_id, target_dir = args.export_risk_outputs
export_outputs(get_hc_id(job_id), expanduser(target_dir), exports)
# import
elif args.load_curve is not None:
with open(args.load_curve) as f:
out = import_hazard_curves(f)
print 'Added output id=%d of type %s; hazard_calculation_id=%d'\
% (out.id, out.output_type, out.oq_job.id)
elif args.list_imported_outputs:
list_imported_outputs()
elif args.delete_uncompleted_calculations:
delete_uncompleted_calculations()
elif args.save_hazard_calculation:
save_hazards.main(*args.save_hazard_calculation)
elif args.load_hazard_calculation:
job_ids = load_hazards.hazard_load(
models.getcursor('admin').connection, args.load_hazard_calculation)
print "Load hazard calculation with IDs: %s" % job_ids
else:
arg_parser.print_usage()
def test_syntax_error(self):
with self.assertRaises(psycopg2.ProgrammingError) as ctx:
upgrade_db(conn, pkg, skip_versions=["0001"])
self.assertTrue(str(ctx.exception).startswith("syntax error at or near"))
def test_missing_pkg(self):
with self.assertRaises(SystemExit) as ctx:
upgrade_db(conn, "openquake.engine.tests.db.not_exists")
self.assertTrue(str(ctx.exception).startswith("Could not import openquake.engine.tests.db.not_exists"))
def test_no_upgrades(self):
with self.assertRaises(SystemExit) as ctx:
upgrade_db(conn, "openquake.engine.tests.db.no_upgrades")
self.assertTrue(str(ctx.exception).startswith("The upgrade_dir does not contain scripts matching the pattern"))
def main():
arg_parser = set_up_arg_parser()
args = arg_parser.parse_args()
if args.version:
print __version__
sys.exit(0)
if args.config_file:
os.environ[config.OQ_CONFIG_FILE_VAR] = \
abspath(expanduser(args.config_file))
config.refresh()
if args.no_distribute:
os.environ[openquake.engine.NO_DISTRIBUTE_VAR] = '1'
if args.upgrade_db:
logging.basicConfig(level=logging.INFO)
logs.set_level('info')
conn = models.getcursor('admin').connection
msg = upgrade_manager.what_if_I_upgrade(
conn, extract_scripts='read_scripts')
print msg
if msg.startswith('Your database is already updated'):
pass
elif args.yes or confirm('Proceed? (y/n) '):
upgrade_manager.upgrade_db(conn)
sys.exit(0)
if args.version_db:
conn = models.getcursor('admin').connection
print upgrade_manager.version_db(conn)
sys.exit(0)
if args.what_if_I_upgrade:
conn = models.getcursor('admin').connection
print upgrade_manager.what_if_I_upgrade(conn)
sys.exit(0)
if args.list_inputs:
list_inputs(args.list_inputs)
# hazard
elif args.list_hazard_calculations:
list_calculations(models.OqJob.objects)
elif args.list_hazard_outputs is not None:
engine.list_hazard_outputs(args.list_hazard_outputs)
elif args.export_hazard is not None:
output_id, target_dir = args.export_hazard
output_id = int(output_id)
export_hazard(output_id, expanduser(target_dir), args.export_type)
elif args.export_hazard_outputs is not None:
hc_id, target_dir = args.export_hazard_outputs
export_hazard_outputs(int(hc_id), expanduser(target_dir),
args.export_type)
elif args.run_hazard is not None:
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
engine.run_job(expanduser(args.run_hazard), args.log_level,
log_file, args.exports)
elif args.delete_hazard_calculation is not None:
del_haz_calc(args.delete_hazard_calculation, args.yes)
# risk
elif args.list_risk_calculations:
list_calculations(models.RiskCalculation.objects)
elif args.list_risk_outputs is not None:
engine.list_risk_outputs(args.list_risk_outputs)
elif args.export_risk is not None:
output_id, target_dir = args.export_risk
export_risk(output_id, expanduser(target_dir), args.export_type)
elif args.export_risk_outputs is not None:
rc_id, target_dir = args.export_risk_outputs
export_risk_outputs(int(rc_id), expanduser(target_dir),
args.export_type)
elif args.run_risk is not None:
if (args.hazard_output_id is None
and args.hazard_calculation_id is None):
sys.exit(MISSING_HAZARD_MSG)
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
engine.run_job(expanduser(args.run_risk), args.log_level, log_file,
args.exports, hazard_output_id=args.hazard_output_id,
hazard_calculation_id=args.hazard_calculation_id)
elif args.delete_risk_calculation is not None:
del_risk_calc(args.delete_risk_calculation, args.yes)
# import
elif args.load_gmf is not None:
with open(args.load_gmf) as f:
out = import_gmf_scenario(f)
print 'Added output id=%d of type %s; hazard_calculation_id=%d'\
% (out.id, out.output_type, out.oq_job.id)
elif args.load_curve is not None:
with open(args.load_curve) as f:
out = import_hazard_curves(f)
print 'Added output id=%d of type %s; hazard_calculation_id=%d'\
% (out.id, out.output_type, out.oq_job.id)
elif args.list_imported_outputs:
list_imported_outputs()
elif args.delete_uncompleted_calculations:
delete_uncompleted_calculations()
elif args.save_hazard_calculation:
save_hazards.main(*args.save_hazard_calculation)
elif args.load_hazard_calculation:
hc_ids = load_hazards.hazard_load(
models.getcursor('admin').connection, args.load_hazard_calculation)
print "Load hazard calculation with IDs: %s" % hc_ids
else:
arg_parser.print_usage()
def test_syntax_error(self):
with self.assertRaises(psycopg2.ProgrammingError) as ctx:
upgrade_db(conn, pkg, skip_versions=['0001'])
self.assertTrue(
str(ctx.exception).startswith('syntax error at or near'))
def main():
arg_parser = set_up_arg_parser()
args = arg_parser.parse_args()
exports = args.exports or 'xml,csv'
if args.version:
print __version__
sys.exit(0)
if args.run or args.run_hazard or args.run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if args.config_file:
os.environ[config.OQ_CONFIG_FILE_VAR] = \
abspath(expanduser(args.config_file))
config.refresh()
if args.no_distribute:
os.environ[openquake.engine.NO_DISTRIBUTE_VAR] = '1'
if args.make_html_report:
conn = models.getcursor('admin').connection
print 'Written', make_report(conn, args.make_html_report)
sys.exit(0)
if args.upgrade_db:
logs.set_level('info')
conn = models.getcursor('admin').connection
msg = upgrade_manager.what_if_I_upgrade(conn,
extract_scripts='read_scripts')
print msg
if msg.startswith('Your database is already updated'):
pass
elif args.yes or confirm('Proceed? (y/n) '):
upgrade_manager.upgrade_db(conn)
sys.exit(0)
if args.version_db:
conn = models.getcursor('admin').connection
print upgrade_manager.version_db(conn)
sys.exit(0)
if args.what_if_I_upgrade:
conn = models.getcursor('admin').connection
print upgrade_manager.what_if_I_upgrade(conn)
sys.exit(0)
if args.list_inputs:
list_inputs(args.list_inputs)
# hazard or hazard+risk
elif args.run:
job_inis = map(expanduser, args.run.split(','))
if len(job_inis) not in (1, 2):
sys.exit('%s should be a .ini filename or a pair of filenames '
'separated by a comma' % args.run)
for job_ini in job_inis:
open(job_ini).read() # raise an IOError if the file does not exist
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
if args.lite:
# run hazard and risk together
engine.run_job_lite(job_inis, args.log_level, log_file,
args.exports)
else:
# run hazard
job = engine.run_job(job_inis[0], args.log_level, log_file,
args.exports)
# run risk
if len(job_inis) == 2:
engine.run_job(job_inis[1],
args.log_level,
log_file,
args.exports,
hazard_calculation_id=job.id)
# hazard
elif args.list_hazard_calculations:
list_calculations('hazard')
elif args.run_hazard is not None:
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
engine.run_job(expanduser(args.run_hazard), args.log_level, log_file,
args.exports)
elif args.delete_hazard_calculation is not None:
del_calc(args.delete_hazard_calculation, args.yes)
# risk
elif args.list_risk_calculations:
list_calculations('risk')
elif args.run_risk is not None:
if (args.hazard_output_id is None
and args.hazard_calculation_id is None):
sys.exit(MISSING_HAZARD_MSG)
log_file = expanduser(args.log_file) \
if args.log_file is not None else None
engine.run_job(expanduser(args.run_risk),
args.log_level,
log_file,
args.exports,
hazard_output_id=args.hazard_output_id,
hazard_calculation_id=args.hazard_calculation_id)
elif args.delete_risk_calculation is not None:
del_calc(args.delete_risk_calculation, args.yes)
# export
elif args.list_outputs is not None:
engine.list_outputs(args.list_outputs)
elif args.list_hazard_outputs is not None:
deprecate('--list-hazard-outputs', '--list-outputs')
engine.list_outputs(args.list_hazard_outputs)
elif args.list_risk_outputs is not None:
deprecate('--list-risk-outputs', '--list-outputs')
engine.list_outputs(args.list_risk_outputs)
elif args.export_output is not None:
output_id, target_dir = args.export_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_hazard_output is not None:
deprecate('--export-hazard-output', '--export-output')
output_id, target_dir = args.export_hazard_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_risk_output is not None:
deprecate('--export-hazard-output', '--export-output')
output_id, target_dir = args.export_risk_output
export(int(output_id), expanduser(target_dir), exports)
elif args.export_outputs is not None:
job_id, target_dir = args.export_outputs
export_outputs(int(job_id), expanduser(target_dir), exports)
elif args.export_stats is not None:
job_id, target_dir, output_type = args.export_stats
export_stats(int(job_id), expanduser(target_dir), output_type, exports)
# deprecated
elif args.export_hazard_outputs is not None:
deprecate('--export-hazard-outputs', '--export-outputs')
job_id, target_dir = args.export_hazard_outputs
export_outputs(int(job_id), expanduser(target_dir), exports)
elif args.export_risk_outputs is not None:
deprecate('--export-risk-outputs', '--export-outputs')
job_id, target_dir = args.export_risk_outputs
export_outputs(int(job_id), expanduser(target_dir), exports)
# import
elif args.load_gmf is not None:
with open(args.load_gmf) as f:
out = import_gmf_scenario(f)
print 'Added output id=%d of type %s; hazard_calculation_id=%d'\
% (out.id, out.output_type, out.oq_job.id)
elif args.load_curve is not None:
with open(args.load_curve) as f:
out = import_hazard_curves(f)
print 'Added output id=%d of type %s; hazard_calculation_id=%d'\
% (out.id, out.output_type, out.oq_job.id)
elif args.list_imported_outputs:
list_imported_outputs()
elif args.delete_uncompleted_calculations:
delete_uncompleted_calculations()
elif args.save_hazard_calculation:
save_hazards.main(*args.save_hazard_calculation)
elif args.load_hazard_calculation:
job_ids = load_hazards.hazard_load(
models.getcursor('admin').connection, args.load_hazard_calculation)
print "Load hazard calculation with IDs: %s" % job_ids
else:
arg_parser.print_usage()
def test_duplicated_version(self):
# there are two scripts with version '0001'
with self.assertRaises(DuplicatedVersion):
with temp_script('0001-do-nothing.sql', 'SELECT 1'):
upgrade_db(conn, pkg)
def test_missing_pkg(self):
with self.assertRaises(SystemExit) as ctx:
upgrade_db(conn, 'openquake.engine.tests.db.not_exists')
self.assertTrue(
str(ctx.exception).startswith(
'Could not import openquake.engine.tests.db.not_exists'))
