def save_uploadfile_in_zipfile(upload_file,
upload_filename,
dest_zipfile_name,
dest_filename_in_zip=None):
nzf = ZipFile(dest_zipfile_name, mode='w', compression=ZIP_DEFLATED)
if upload_filename[-3:].lower() == u'zip':
# 'zipfile, check and rename content'
zf = ZipFile(StringIO(upload_file.read()), 'r')
file_names = zf.namelist()
for filename in file_names:
#write file to new
a = get_new_filename(filename, dest_filename_in_zip)
nzf.writestr(a, zf.read(filename))
zf.close()
else:
# 'normal file'
nzf.writestr(get_new_filename(upload_filename, dest_filename_in_zip),
upload_file.read())
nzf.close()
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
def save_uploadfile_in_zipfile(
upload_file, upload_filename,
dest_zipfile_name, dest_filename_in_zip=None):
nzf = ZipFile(dest_zipfile_name, mode='w', **ZIP_WRITE_OPTS)
if upload_filename[-3:].lower() == u'zip':
# 'zipfile, check and rename content'
zf = ZipFile(StringIO(upload_file.read()), 'r')
file_names = zf.namelist()
for filename in file_names:
#write file to new
a = get_new_filename(filename, dest_filename_in_zip)
nzf.writestr(a, zf.read(filename))
zf.close()
else:
# 'normal file'
nzf.writestr(
get_new_filename(upload_filename, dest_filename_in_zip),
upload_file.read())
nzf.close()
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
def save_uploadfile_in_zipfile_groupimport(
upload_zipfile, re_filenames_in_upload_file,
dest_zipfile_name, dest_filename_in_zip=None):
nzf = ZipFile(dest_zipfile_name, mode='w', compression=ZIP_DEFLATED)
zf = upload_zipfile
reg_ex = '([0-9]*)'.join(
[b for b in re_filenames_in_upload_file.split('#') if b != ''])
reg_ex = (reg_ex.replace('\\', '/').replace('+', '\+').
replace('(', '\(').replace(')', '\)'))
reg_ex = re.compile(reg_ex, re.I)
found = False
for filename in zf.namelist():
filename = filename.replace('\\', '/')
# write file to new
if reg_ex.match(filename):
# remove path
new_filename = filename.replace('\\', '/').split('/')[-1]
a = get_new_filename(new_filename, dest_filename_in_zip)
nzf.writestr(a.lower(), zf.read(filename))
found = True
nzf.close()
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
if not found:
raise KeyError('File not found')
def save_uploadfile_in_zipfile_groupimport(upload_zipfile,
re_filenames_in_upload_file,
dest_zipfile_name,
dest_filename_in_zip=None):
nzf = ZipFile(dest_zipfile_name, mode='w', compression=ZIP_DEFLATED)
zf = upload_zipfile
reg_ex = '([0-9]*)'.join(
[b for b in re_filenames_in_upload_file.split('#') if b != ''])
reg_ex = (reg_ex.replace('\\', '/').replace('+', '\+').replace(
'(', '\(').replace(')', '\)'))
reg_ex = re.compile(reg_ex, re.I)
found = False
for filename in zf.namelist():
filename = filename.replace('\\', '/')
# write file to new
if reg_ex.match(filename):
# remove path
new_filename = filename.replace('\\', '/').split('/')[-1]
a = get_new_filename(new_filename, dest_filename_in_zip)
nzf.writestr(a.lower(), zf.read(filename))
found = True
nzf.close()
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
if not found:
raise KeyError('File not found')
def handle(self, *args, **options):
if not args:
raise CommandError("No path given.")
path = args[0]
if not os.path.exists(path) or not os.path.isdir(path):
raise CommandError("'{0}' is not an existing directory.".
format(path))
remove_comments_from_asc_files(path, verbose=True)
def save_uploadfile_in_zipfile_groupimport(upload_zipfile,
re_filenames_in_upload_file,
dest_zipfile_name,
dest_filename_in_zip=None):
reg_ex = '([0-9]*)'.join(
[b for b in re_filenames_in_upload_file.split('#') if b != ''])
reg_ex = (reg_ex.replace('\\', '/').replace('+', '\+').replace(
'(', '\(').replace(')', '\)'))
reg_ex = re.compile(reg_ex, re.I)
found = False
for filename in upload_zipfile.namelist():
filename = filename.replace('\\', '/')
# write file to new
if reg_ex.match(filename):
# remove path
new_filename = filename.replace('\\', '/').split('/')[-1]
target_item = get_new_filename(new_filename, dest_filename_in_zip)
# zip crashes on existing empty destination files
if os.path.exists(dest_zipfile_name):
os.remove(dest_zipfile_name)
rezip(
source_path=upload_zipfile.filename,
source_item=filename,
target_path=dest_zipfile_name,
target_item=target_item,
)
found = True
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
if not found:
raise KeyError('File not found')
def save_uploadfile_in_zipfile_groupimport(
upload_zipfile, re_filenames_in_upload_file,
dest_zipfile_name, dest_filename_in_zip=None):
reg_ex = '([0-9]*)'.join(
[b for b in re_filenames_in_upload_file.split('#') if b != ''])
reg_ex = (reg_ex.replace('\\', '/').replace('+', '\+').
replace('(', '\(').replace(')', '\)'))
reg_ex = re.compile(reg_ex, re.I)
found = False
for filename in upload_zipfile.namelist():
filename = filename.replace('\\', '/')
# write file to new
if reg_ex.match(filename):
# remove path
new_filename = filename.replace('\\', '/').split('/')[-1]
target_item = get_new_filename(new_filename, dest_filename_in_zip)
# zip crashes on existing empty destination files
if os.path.exists(dest_zipfile_name):
os.remove(dest_zipfile_name)
rezip(
source_path=upload_zipfile.filename,
source_item=filename,
target_path=dest_zipfile_name,
target_item=target_item,
)
found = True
# Remove comment line from .asc and .inc files after uploading
remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
if not found:
raise KeyError('File not found')
def perform_sobek_simulation(scenario_id,
sobek_project_directory,
sobek_program_root,
project_name='lizardkb',
timeout=288000):
"""task 130: perform_sobek_simulation
"""
log.debug("step 0a: get settings")
log.debug("sobek_project_directory: %s" % sobek_project_directory)
log.debug("sobek_program_root: %s" % sobek_program_root)
scenario = Scenario.objects.get(pk=scenario_id)
sobek_version = scenario.sobekmodel_inundation.sobekversion.name[:5]
sobek_location = os.path.join(
sobek_program_root,
default_sobek_locations[sobek_version])
#sobek_location = [d for d in sobek_location.split('/') if d]
log.debug("sobek_location: %s" % sobek_location)
destination_dir = Setting.objects.get(key='DESTINATION_DIR').value
source_dir = Setting.objects.get(key='SOURCE_DIR').value
project_dir = os.path.join(sobek_location, sobek_project_directory)
log.debug("project_dir: %s" % project_dir)
log.debug("compute the local location of sobek files")
# first keep all paths as lists of elements, will join them using
# os.sep at the latest possible moment.
case_1_dir = os.path.join(project_dir, '1')
work_dir = os.path.join(project_dir, 'WORK')
cmtwork_dir = os.path.join(project_dir, 'CMTWORK')
output_dir_name = os.path.join(destination_dir, scenario.get_rel_destdir())
model_file_location = os.path.join(
destination_dir, scenario.result_set.get(resulttype=26).resultloc)
log.debug("empty project_dir WORK & 1")
for to_empty in [work_dir, case_1_dir, cmtwork_dir]:
for root, dirs, files in os.walk(to_empty):
for name in files:
os.remove(os.path.join(root, name))
log.debug("open the archived sobek model " + output_dir_name + "model.zip")
input_file = ZipFile(model_file_location, "r")
log.debug("unpacking the archived sobek model to project_dir WORK & 1")
if not os.path.isdir(os.path.join(work_dir, 'grid')):
os.makedirs(os.path.join(work_dir, 'grid'))
if not os.path.isdir(os.path.join(case_1_dir, 'grid')):
os.makedirs(os.path.join(case_1_dir, 'grid'))
for name in input_file.namelist():
content = input_file.read(name)
temp = file(os.path.join(work_dir, name), "wb")
temp.write(content)
temp.close()
temp = file(os.path.join(case_1_dir, name), "wb")
temp.write(content)
temp.close()
settings_ini_location = os.path.join(
source_dir,
scenario.sobekmodel_inundation.sobekversion.fileloc_startfile)
log.debug("copy from " + settings_ini_location + " to the CMTWORK dir")
for name in ['simulate.ini', 'casedesc.cmt']:
temp = file(os.path.join(cmtwork_dir, name), "w")
content = file(os.path.join(settings_ini_location, name), "r").read()
content = content.replace('lizardkb.lit', sobek_project_directory)
content = content.replace('LIZARDKB.LIT', sobek_project_directory)
temp.write(content)
temp.close()
program_name = os.path.join(sobek_location, "programs", "simulate.exe")
configuration = os.path.join(cmtwork_dir, 'simulate.ini')
log.debug("Close connection before spawning a subprocess.")
db.close_connection()
log.debug('about to spawn the simulate subprocess')
cmd, cwd = [program_name, configuration], cmtwork_dir
log.debug('command_list: %s, current_dir: %s' % (cmd, cwd))
os.chdir(cwd)
child = subprocess.Popen(cmd)
log.debug('about to start the watchdog thread')
log.debug('cmtwork_dir %s, %s' % (type(cmtwork_dir), cmtwork_dir))
watchdog_t = threading.Thread(target=watchdog, args=(child, cmtwork_dir))
watchdog_t.start()
log.debug('about to start the alarm thread')
alarm_t = threading.Thread(target=alarm_handler, args=(timeout, child,))
alarm_t.start()
log.debug("starting to wait for completion of subprocess")
child.wait()
log.debug('child returned')
log.debug('save results to {0}'.format(output_dir_name))
resulttypes = ResultType.objects.filter(program=1)
max_file_nr, min_file_nr = save_output_files_to_dest(
output_dir_name, work_dir, resulttypes)
save_results_to_db(resulttypes, max_file_nr, min_file_nr, scenario)
# remove commentline from asc-files, inc-files
remove_comments_from_asc_files(output_dir_name,
max_uncompressed_zip_size=None)
log.debug("check return code and return False if not ok")
try:
output = file(os.path.join(cmtwork_dir, 'PLUVIUS1.rtn'), "r")
remarks = output.read()
except:
remarks = ' 51\nerror reading output file'
remarks = 'rev: ' + __revision__ + "\n" + remarks
log.info(remarks)
log.debug("close db connection to avoid an idle process.")
db.close_connection()
successful = int(re.findall(r'\d+', remarks)[0]) == 0
return successful
def perform_sobek_simulation(scenario_id,
sobek_project_directory,
sobek_program_root,
project_name='lizardkb',
timeout=288000):
"""task 130: perform_sobek_simulation
"""
log.debug("step 0a: get settings")
log.debug("sobek_project_directory: %s" % sobek_project_directory)
log.debug("sobek_program_root: %s" % sobek_program_root)
scenario = Scenario.objects.get(pk=scenario_id)
sobek_version = scenario.sobekmodel_inundation.sobekversion.name[:5]
sobek_location = os.path.join(sobek_program_root,
default_sobek_locations[sobek_version])
#sobek_location = [d for d in sobek_location.split('/') if d]
log.debug("sobek_location: %s" % sobek_location)
destination_dir = settings.EXTERNAL_RESULT_MOUNTED_DIR
source_dir = settings.EXTERNAL_SOURCE_MOUNTED_DIR
project_dir = os.path.join(sobek_location, sobek_project_directory)
log.debug("project_dir: %s" % project_dir)
log.debug("compute the local location of sobek files")
# first keep all paths as lists of elements, will join them using
# os.sep at the latest possible moment.
case_1_dir = os.path.join(project_dir, '1')
work_dir = os.path.join(project_dir, 'WORK')
cmtwork_dir = os.path.join(project_dir, 'CMTWORK')
output_dir_name = os.path.join(destination_dir, scenario.get_rel_destdir())
model_file_location = os.path.join(
destination_dir,
scenario.result_set.get(resulttype=26).resultloc)
log.debug("empty project_dir WORK & 1")
for to_empty in [work_dir, case_1_dir, cmtwork_dir]:
for root, dirs, files in os.walk(to_empty):
for name in files:
os.remove(os.path.join(root, name))
log.debug("open the archived sobek model " + output_dir_name + "model.zip")
input_file = ZipFile(model_file_location, "r")
log.debug("unpacking the archived sobek model to project_dir WORK & 1")
if not os.path.isdir(os.path.join(work_dir, 'grid')):
os.makedirs(os.path.join(work_dir, 'grid'))
if not os.path.isdir(os.path.join(case_1_dir, 'grid')):
os.makedirs(os.path.join(case_1_dir, 'grid'))
for name in input_file.namelist():
content = input_file.read(name)
temp = file(os.path.join(work_dir, name), "wb")
temp.write(content)
temp.close()
temp = file(os.path.join(case_1_dir, name), "wb")
temp.write(content)
temp.close()
settings_ini_location = os.path.join(
source_dir,
scenario.sobekmodel_inundation.sobekversion.fileloc_startfile)
log.debug("copy from " + settings_ini_location + " to the CMTWORK dir")
for name in ['simulate.ini', 'casedesc.cmt']:
temp = file(os.path.join(cmtwork_dir, name), "w")
content = file(os.path.join(settings_ini_location, name), "r").read()
content = content.replace('lizardkb.lit', sobek_project_directory)
content = content.replace('LIZARDKB.LIT', sobek_project_directory)
temp.write(content)
temp.close()
program_name = os.path.join(sobek_location, "programs", "simulate.exe")
configuration = os.path.join(cmtwork_dir, 'simulate.ini')
log.debug("Close connection before spawning a subprocess.")
db.close_connection()
log.debug('about to spawn the simulate subprocess')
cmd, cwd = [program_name, configuration], cmtwork_dir
log.debug('command_list: %s, current_dir: %s' % (cmd, cwd))
os.chdir(cwd)
child = subprocess.Popen(cmd)
log.debug('about to start the watchdog thread')
log.debug('cmtwork_dir %s, %s' % (type(cmtwork_dir), cmtwork_dir))
watchdog_t = threading.Thread(target=watchdog, args=(child, cmtwork_dir))
watchdog_t.start()
log.debug('about to start the alarm thread')
alarm_t = threading.Thread(target=alarm_handler, args=(
timeout,
child,
))
alarm_t.start()
log.debug("starting to wait for completion of subprocess")
child.wait()
log.debug('child returned')
log.debug('save results to {0}'.format(output_dir_name))
resulttypes = ResultType.objects.filter(program=1)
max_file_nr, min_file_nr = save_output_files_to_dest(
output_dir_name, work_dir, resulttypes)
save_results_to_db(resulttypes, max_file_nr, min_file_nr, scenario)
# remove commentline from asc-files, inc-files
remove_comments_from_asc_files(output_dir_name,
max_uncompressed_zip_size=None)
log.debug("check return code and return False if not ok")
try:
output = file(os.path.join(cmtwork_dir, 'PLUVIUS1.rtn'), "r")
remarks = output.read()
except:
remarks = ' 51\nerror reading output file'
remarks = 'rev: ' + __revision__ + "\n" + remarks
log.info(remarks)
log.debug("close db connection to avoid an idle process.")
db.close_connection()
successful = int(re.findall(r'\d+', remarks)[0]) == 0
return successful
