def process_threedi_nc(
some_id, some_type,
detailed=True, with_region=True,
gridsize=None,
gridsize_divider=2):
"""
Read netcdf (.nc) 3Di result file, output in .png files.
Input:
There must be a "subgrid_map.zip" (and a
scenario.zip) at (scenario.get_abs_destdir())/threedi.
Output:
folder with png files and a Result object corresponding to this
folder.
"""
region = None
scenario_id = some_id
scenario = Scenario.objects.get(pk=scenario_id)
try:
if with_region:
region = scenario.breaches.all()[0].region
except:
print 'Something went wrong with getting region extent for scenario %s' % scenario
#full_path = "/home/jack/git/sites/flooding/driedi/Vecht/subgrid_map.nc"
full_base_path = scenario.get_abs_destdir().replace('\\', '/')
full_path_zip = os.path.join(
full_base_path, 'threedi', 'subgrid_map.zip')
result_folder = os.path.join(full_base_path, 'threedi_waterlevel_png')
#print result_folder
try:
print 'creating result folder...'
os.makedirs(result_folder)
except OSError:
print 'warning: error creating folder %s, does it already exist?' % result_folder
dst_basefilename = os.path.join(result_folder, 'waterlevel_%04d_j')
dst_basefilename_detailed = os.path.join(result_folder, 'waterlevel_%04d')
print 'input: %s' % full_path_zip
if detailed:
print 'output (detailed): %s' % dst_basefilename_detailed
else:
print 'output: %s' % dst_basefilename
with files.unzipped(full_path_zip) as files_in_zip:
for filename in files_in_zip:
if filename.endswith('subgrid_map.nc'):
print 'yup, subgrid_map.nc found, proceeding'
if detailed:
num_steps = post_process_detailed_3di(
filename, dst_basefilename_detailed,
region=region, gridsize_divider=gridsize_divider,
gridsize=gridsize)
else:
num_steps = post_process_3di(filename, dst_basefilename)
else:
print 'ignored file %s' % filename
#print 'results are available at:'
#print filenames
print 'creating corresponding Result object...'
src_rel_path = os.path.join(
scenario.get_rel_destdir(),
'threedi',
'subgrid_map.zip')
dst_rel_path = os.path.join(
scenario.get_rel_destdir(),
'threedi_waterlevel_png',
'waterlevel_####.png')
result_type = ResultType.objects.get(name='threediwaterlevel_t')
result, created = Result.objects.get_or_create(
resulttype=result_type,
scenario=scenario,
defaults={
'resultloc': src_rel_path,
'resultpngloc': dst_rel_path,
'startnr': 0,
'firstnr': 0,
'lastnr': num_steps-1}
)
if not created:
print 'updating existing result.'
print 'old results: %s %s' % (result.resultloc, result.resultpngloc)
if result.resultpngloc != dst_rel_path or result.resultloc != src_rel_path:
print '!delete old result files manually!'
result.resultloc = src_rel_path
result.resultpngloc = dst_rel_path
result.startnr = 0
result.firstnr = 0
result.lastnr = num_steps - 1
print 'new results: %s %s' % (result.resultloc, result.resultpngloc)
result.save()
print 'result updated/created with id %d' % result.id
def run_threedi_task(some_id, some_type):
"""
3Di task
some_id and some_type:
if some_type = 'threedi_calculation_id', then some_id is the
threedi_calculation_id, otherwise it is the scenario_id.
SobekModel can also be a 3di model:
- project_fileloc contains the .zip filename
- model_varname contains the .mdu filename
"""
if hasattr(settings, 'THREEDI_DEFAULT_FILES_PATH'):
default_source_files_path = settings.THREEDI_DEFAULT_FILES_PATH
else:
default_source_files_path = "/home/jack/3di-subgrid/bin"
print 'No THREEDI_DEFAULT_FILES_PATH in django settings. Taking default %s' % default_source_files_path
if hasattr(settings, 'THREEDI_BIN_PATH'):
subgrid_exe_path = settings.THREEDI_BIN_PATH
else:
subgrid_exe_path = "/home/jack/3di-subgrid/bin/subgridf90"
print 'No THREEDI_BIN_PATH in django settings. Taking default %s' % subgrid_exe_path
scenario_id = some_id
scenario = Scenario.objects.get(pk=scenario_id)
print 'Working on 3Di calculation: %s (id %d)' % (scenario, scenario.id)
if str(scenario.sobekmodel_inundation.sobekversion) != '3di':
print 'Warning: "sobekversion" is not "3di", but "%s". Is "%s" a 3di model?' % (
scenario.sobekmodel_inundation.sobekversion,
scenario.sobekmodel_inundation)
full_path = full_model_path(scenario.sobekmodel_inundation.project_fileloc)
print 'Unzipping %s...' % full_path
with files.unzipped(full_path) as files_in_zip:
tmp_path = os.path.dirname(
files_in_zip[0]) # Assume no subdirs or whatsoever
mdu_full_path = os.path.join(
tmp_path, scenario.sobekmodel_inundation.model_varname)
#print files_in_zip
#nc_filename = setup_and_run_3di(mdu_full_path, skip_if_results_available=False)
print "configuring mdu file (put config_3di in it)..."
configure_3di(mdu_full_path, scenario)
print "running..."
nc_filename = setup_and_run_3di(
mdu_full_path,
skip_if_results_available=True,
source_files_dir=default_source_files_path,
subgrid_exe=subgrid_exe_path)
result_folder = os.path.join(
scenario.get_abs_destdir().replace('\\', '/'), 'threedi')
#threedi_calculation.full_result_path
#print result_folder
try:
print 'creating result folder %s...' % result_folder
os.makedirs(result_folder)
except OSError:
print 'warning: error creating folder %s, does it already exist?' % result_folder
# make subgrid_map.zip from subgrid_map.nc
tmp_nc_zip = files.mktemp()
files.add_to_zip(tmp_nc_zip, [{
'filename': nc_filename,
'arcname': 'subgrid_map.nc'
}])
try:
print 'copying nc zip file...'
shutil.copy(tmp_nc_zip,
os.path.join(result_folder, 'subgrid_map.zip'))
os.remove(tmp_nc_zip)
except:
print 'problem copying nc zip file %s to %s' % (tmp_nc_zip,
result_folder)
# # move .nc to result folder
# try:
# print 'moving nc file...'
# shutil.move(nc_filename, result_folder)
# except:
# print 'problem moving nc file %s to %s' % (nc_filename, result_folder)
# zip the other files to result folder
tmp_zipfile = files.mktemp()
for root, dirs, filenames in os.walk(tmp_path):
for filename in filenames:
files.add_to_zip(tmp_zipfile,
[{
'filename': os.path.join(root, filename),
'arcname': filename,
'delete_after': True
}])
#print tmp_zipfile
shutil.copy(tmp_zipfile, os.path.join(result_folder, 'scenario.zip'))
os.remove(tmp_zipfile)
print 'Finished.'
def run_threedi_task(some_id, some_type):
"""
3Di task
some_id and some_type:
if some_type = 'threedi_calculation_id', then some_id is the
threedi_calculation_id, otherwise it is the scenario_id.
SobekModel can also be a 3di model:
- project_fileloc contains the .zip filename
- model_varname contains the .mdu filename
"""
if hasattr(settings, 'THREEDI_DEFAULT_FILES_PATH'):
default_source_files_path = settings.THREEDI_DEFAULT_FILES_PATH
else:
default_source_files_path = "/home/jack/3di-subgrid/bin"
print 'No THREEDI_DEFAULT_FILES_PATH in django settings. Taking default %s' % default_source_files_path
if hasattr(settings, 'THREEDI_BIN_PATH'):
subgrid_exe_path = settings.THREEDI_BIN_PATH
else:
subgrid_exe_path = "/home/jack/3di-subgrid/bin/subgridf90"
print 'No THREEDI_BIN_PATH in django settings. Taking default %s' % subgrid_exe_path
scenario_id = some_id
scenario = Scenario.objects.get(pk=scenario_id)
print 'Working on 3Di calculation: %s (id %d)' % (scenario, scenario.id)
if str(scenario.sobekmodel_inundation.sobekversion) != '3di':
print 'Warning: "sobekversion" is not "3di", but "%s". Is "%s" a 3di model?' % (
scenario.sobekmodel_inundation.sobekversion, scenario.sobekmodel_inundation)
full_path = full_model_path(
scenario.sobekmodel_inundation.project_fileloc)
print 'Unzipping %s...' % full_path
with files.unzipped(full_path) as files_in_zip:
tmp_path = os.path.dirname(files_in_zip[0]) # Assume no subdirs or whatsoever
mdu_full_path = os.path.join(tmp_path, scenario.sobekmodel_inundation.model_varname)
#print files_in_zip
#nc_filename = setup_and_run_3di(mdu_full_path, skip_if_results_available=False)
print "running..."
nc_filename = setup_and_run_3di(
mdu_full_path,
skip_if_results_available=True,
source_files_dir=default_source_files_path,
subgrid_exe=subgrid_exe_path)
result_folder = os.path.join(scenario.get_abs_destdir().replace('\\', '/'), 'threedi')
#threedi_calculation.full_result_path
#print result_folder
try:
print 'creating result folder %s...' % result_folder
os.makedirs(result_folder)
except OSError:
print 'warning: error creating folder %s, does it already exist?' % result_folder
# make subgrid_map.zip from subgrid_map.nc
tmp_nc_zip = files.mktemp()
files.add_to_zip(
tmp_nc_zip,
[{'filename': nc_filename, 'arcname': 'subgrid_map.nc'}])
try:
print 'copying nc zip file...'
shutil.copy(tmp_nc_zip, os.path.join(result_folder, 'subgrid_map.zip'))
os.remove(tmp_nc_zip)
except:
print 'problem copying nc zip file %s to %s' % (tmp_nc_zip, result_folder)
# # move .nc to result folder
# try:
# print 'moving nc file...'
# shutil.move(nc_filename, result_folder)
# except:
# print 'problem moving nc file %s to %s' % (nc_filename, result_folder)
# zip the other files to result folder
tmp_zipfile = files.mktemp()
for root, dirs, filenames in os.walk(tmp_path):
for filename in filenames:
files.add_to_zip(tmp_zipfile, [{
'filename': os.path.join(root, filename),
'arcname': filename,
'delete_after': True
}])
#print tmp_zipfile
shutil.copy(tmp_zipfile, os.path.join(result_folder, 'scenario.zip'))
os.remove(tmp_zipfile)
print 'Finished.'
def process_threedi_nc(some_id,
some_type,
detailed=True,
with_region=True,
gridsize=None,
gridsize_divider=2):
"""
Read netcdf (.nc) 3Di result file, output in .png files.
Input:
There must be a "subgrid_map.zip" (and a
scenario.zip) at (scenario.get_abs_destdir())/threedi.
Output:
folder with png files and a Result object corresponding to this
folder.
"""
region = None
scenario_id = some_id
scenario = Scenario.objects.get(pk=scenario_id)
try:
if with_region:
region = scenario.breaches.all()[0].region
except:
print 'Something went wrong with getting region extent for scenario %s' % scenario
#full_path = "/home/jack/git/sites/flooding/driedi/Vecht/subgrid_map.nc"
full_base_path = scenario.get_abs_destdir().replace('\\', '/')
full_path_zip = os.path.join(full_base_path, 'threedi', 'subgrid_map.zip')
result_folder = os.path.join(full_base_path, 'threedi_waterlevel_png')
#print result_folder
try:
print 'creating result folder...'
os.makedirs(result_folder)
except OSError:
print 'warning: error creating folder %s, does it already exist?' % result_folder
dst_basefilename = os.path.join(result_folder, 'waterlevel_%04d_j')
dst_basefilename_detailed = os.path.join(result_folder, 'waterlevel_%04d')
print 'input: %s' % full_path_zip
if detailed:
print 'output (detailed): %s' % dst_basefilename_detailed
else:
print 'output: %s' % dst_basefilename
with files.unzipped(full_path_zip) as files_in_zip:
for filename in files_in_zip:
if filename.endswith('subgrid_map.nc'):
print 'yup, subgrid_map.nc found, proceeding'
if detailed:
num_steps = post_process_detailed_3di(
filename,
dst_basefilename_detailed,
region=region,
gridsize_divider=gridsize_divider,
gridsize=gridsize)
else:
num_steps = post_process_3di(filename, dst_basefilename)
else:
print 'ignored file %s' % filename
#print 'results are available at:'
#print filenames
print 'creating corresponding Result object...'
src_rel_path = os.path.join(scenario.get_rel_destdir(), 'threedi',
'subgrid_map.zip')
dst_rel_path = os.path.join(scenario.get_rel_destdir(),
'threedi_waterlevel_png',
'waterlevel_####.png')
result_type = ResultType.objects.get(name='threediwaterlevel_t')
result, created = Result.objects.get_or_create(resulttype=result_type,
scenario=scenario,
defaults={
'resultloc':
src_rel_path,
'resultpngloc':
dst_rel_path,
'startnr': 0,
'firstnr': 0,
'lastnr': num_steps - 1
})
if not created:
print 'updating existing result.'
print 'old results: %s %s' % (result.resultloc, result.resultpngloc)
if result.resultpngloc != dst_rel_path or result.resultloc != src_rel_path:
print '!delete old result files manually!'
result.resultloc = src_rel_path
result.resultpngloc = dst_rel_path
result.startnr = 0
result.firstnr = 0
result.lastnr = num_steps - 1
print 'new results: %s %s' % (result.resultloc, result.resultpngloc)
result.save()
print 'result updated/created with id %d' % result.id