def results_home(request):
from Results.csv_generator import SUMMARY_FILE_NAME
path_ex = workspace_path(scenario_filename() +"/*.csv")
start = workspace_path()
context = {'supplemental_files': [os.path.relpath(file_path, start=start) for file_path in glob(path_ex)]}
summary_path = os.path.join(scenario_filename(), SUMMARY_FILE_NAME)
try:
context['supplemental_files'].remove(summary_path)
# context['supplemental_files'] = [file for file in context['supplemental_files'] if not file.endswith(SUMMARY_FILE_NAME)] # filter out summary.csv
except ValueError: pass
context['summary_file_name'] = summary_path
if os.path.exists(map_zip_file()):
context['supplemental_files'].append(os.path.relpath(map_zip_file(), start=start))
# TODO: value dict file sizes
if DailyControls.objects.all().count() > 0:
context['summary'] = Results.summary.summarize_results()
context['iterations'] = len(list_of_iterations())
context['population_eta'] = Unit.objects.count() / 650 # estimate slow map calc in matplotlib
try:
v = ResultsVersion.objects.get()
context['version_number'] = '.'.join([v.versionMajor, v.versionMinor, v.versionRelease])
except: # more specific exceptions kept leaking through
pass
return render(request, 'Results/SimulationProgress.html', context)
def make_population_map_file(self):
if not os.path.exists(workspace_path(scenario_filename())):
os.makedirs(workspace_path(scenario_filename()))
print("Calculating a new Population Map")
fig = population_results_map()
FigureCanvas(fig).print_png(self.path)
thumbnail(self.path, self.thumb_path, scale=0.1923)
print("Finished Population Map")
def copy_file(request, target, destination):
if target.replace('.sqlite3', '') == scenario_filename(): # copying the active scenario
return save_scenario(request)
if not destination.endswith('.sqlite3'):
destination += ".sqlite3"
print("Copying", target, "to", destination, ". This could take several minutes...")
shutil.copy(workspace_path(target), workspace_path(destination))
print("Done copying", target)
return redirect('/')
def population_thumbnail_png(request, second_try=False):
path = workspace_path(scenario_filename() + '/population_map.png')
thumb_path = workspace_path(scenario_filename() + '/population_thumbnail.png')
try:
with open(thumb_path, "rb") as f:
return HttpResponse(f.read(), content_type="image/png")
except IOError:
if os.path.exists(path):
if second_try:
sleep(1)
thumbnail(path, thumb_path, scale=0.1923) # create the thumbnail
return population_thumbnail_png(request, second_try=True)
else:
sleep(5)
return population_thumbnail_png(request, second_try=False)
def test_post_with_points_from_file(self):
points_file = tempfile.NamedTemporaryFile(delete=False)
points_file.write(b"x,y\n0.0, 1.0\n")
points_file.seek(0)
form_data = {
'relationalpoint_set-TOTAL_FORMS': '0',
'relationalpoint_set-INITIAL_FORMS': '0',
'relationalpoint_set-MAX_NUM_FORMS': '1',
'name': 'Test Function',
'x_axis_units': 'Days',
'file': points_file
}
r = self.client.post('/setup/RelationalFunction/new/', form_data, follow=True)
temp_file_name = os.path.basename(points_file.name)
temp_file_upload_path = workspace_path(temp_file_name)
self.assertFalse(os.path.exists(temp_file_upload_path))
self.assertEqual(RelationalFunction.objects.count(), 1)
function = RelationalFunction.objects.first()
self.assertEqual(function.name, 'Test Function')
self.assertEqual(RelationalPoint.objects.count(), 1)
point = RelationalPoint.objects.first()
self.assertEqual(point.x, 0.0)
self.assertEqual(point.y, 1.0)
self.assertIn('/setup/RelationalFunction/%d/' % function.pk, r.content.decode())
points_file.close()
os.unlink(points_file.name)
def download_file(request):
target = request.GET['target']
target = target if target[-1] not in r'/\\' else target[:-1] # shouldn't be a trailing slash
file_path = workspace_path(target)
f = open(file_path, "rb")
response = HttpResponse(f, content_type="application/x-sqlite") # TODO: generic content type
response['Content-Disposition'] = 'attachment; filename="' + target
return response
def run(self):
if self.testing:
for database in settings.DATABASES:
settings.DATABASES[database]['NAME'] = settings.DATABASES[database]['TEST']['NAME'] if 'TEST' in settings.DATABASES[database] else settings.DATABASES[database]['TEST_NAME']
location = workspace_path(scenario_filename() + '/' + SUMMARY_FILE_NAME) # Note: scenario_filename uses the database
headers, data = self.get_summary_data_table()
create_csv_file(location, headers, data)
def population_zoom_png(request=None):
path = workspace_path(scenario_filename() + '/population_map.png')
thumb_path = workspace_path(scenario_filename() + '/population_thumbnail.png')
try:
with open(path, "rb") as img_file:
return HttpResponse(img_file.read(), content_type='image/png')
except IOError:
save_image = is_simulation_stopped() # we want to check this before reading the stats, this is in motion
if not save_image: # in order to avoid database locked Issue #150
return population_png(request, 58.5, 52)
else:
if any(thread.name == 'population_map_thread' for thread in threading.enumerate()):
print("Waiting on a Population Map")
sleep(5)
else:
PopulationWorker(path, thumb_path).start()
sleep(.5)
return population_zoom_png(request)
def delete_supplemental_folder():
scenario_folder = scenario_filename()
if scenario_folder != '':
try:
shutil.rmtree(workspace_path(scenario_folder))
except:
pass # because the folder doesn't exist (which is fine)
from django.db import connections
connections['scenario_db'].cursor().execute('VACUUM')
def zip_map_directory_if_it_exists():
dir_to_zip = workspace_path(scenario_filename() + "/Map")
if os.path.exists(dir_to_zip) and supplemental_folder_has_contents(subfolder='/Map'):
zipname = map_zip_file()
dir_to_zip_len = len(dir_to_zip.rstrip(os.sep)) + 1
with zipfile.ZipFile(zipname, mode='w', compression=zipfile.ZIP_DEFLATED) as zf:
for dirname, subdirs, files in os.walk(dir_to_zip):
for filename in files:
path = os.path.join(dirname, filename)
entry = path[dir_to_zip_len:]
zf.write(path, entry)
else:
print("Folder is empty: ", dir_to_zip)
def test_save_scenario_success(self):
filename_field = self.save_scenario_as()
try:
filename_field.send_keys('123.1 AZ')
self.find('.modal.in .btn-primary').click()
time.sleep(3)
status = self.find('.scenario-status')
self.assertNotIn('unsaved', status.get_attribute('class'))
finally:
try:
os.remove(workspace_path('Untitled Scenario123.1 AZ.sqlite3'))
except:
pass
def test_post_failure_bad_xml(self):
expected_results = {
'status': 'failed',
'message': "This is not a valid Population file: "+'mismatched tag: line 17, column 2'
}
try:
os.remove(workspace_path('Population_Test_Invalid.xml'),)
except OSError: pass
with open(POPULATION_FIXTURES + 'Population_Test_Invalid.xml', mode='rb') as fp:
r = self.client.post('/setup/UploadPopulation/', {'file': fp})
# data = json.loads(r.content.decode())
self.assertJSONEqual(r.content.decode(), expected_results)
def open_scenario(request, target, wrap_target=True):
if wrap_target:
target = workspace_path(target)
print("Copying ", target, "to", db_path(), ". This could take several minutes...")
close_old_connections()
shutil.copy(target, db_path(name='scenario_db'))
scenario_filename(os.path.basename(target))
print('Sessions overwritten with ', target)
update_db_version()
unsaved_changes(False) # File is now in sync
SmSession.objects.all().update(iteration_text = '', simulation_has_started=outputs_exist()) # This is also reset from delete_all_outputs
# else:
# print('File does not exist')
return redirect('/setup/Scenario/1/')
def handle(self, *args, **options):
print("Preparing to migrate all Scenario Databases to the current state...")
current_scenario = db_path(name='scenario_db')
current_settings = db_path(name='default')
moved_current = False
try:
shutil.move(current_scenario, current_scenario + '.tmp')
shutil.move(current_settings, current_settings + '.tmp')
moved_current = True
print("Backed up current Active Session.")
except:
print("Failed to backup current Active Session! Note: it may just not exist which is fine.")
graceful_startup()
locations = [os.path.join(settings.BASE_DIR, 'ScenarioCreator', 'tests', 'population_fixtures'), os.path.join(settings.BASE_DIR, 'Sample Scenarios'), ]
if not options['skip_workspace']:
locations.append(workspace_path())
print("\nIncluding User's Workspace folder.")
else:
print("\nExcluding User's Workspace folder.")
connections['scenario_db'].close()
close_old_connections()
for location in locations:
for root, dirs, files in os.walk(location):
for file in files:
if file.endswith(".sqlite3") and file != "settings.sqlite3":
print("\nMigrating", file, "in", root + "...")
try:
open_test_scenario(request=None, target=os.path.join(root, file))
connections['scenario_db'].close()
close_old_connections()
shutil.move(db_path('scenario_db'), os.path.join(root, file))
print("\nDone.")
except Exception as e:
print("\nFailed to migrate", file + "!", e)
continue
try:
if moved_current:
shutil.move(current_scenario + '.tmp', current_scenario)
shutil.move(current_settings + '.tmp', current_settings)
print("Reverted current Active Session.")
except:
print("Failed to revert current Active Session!")
print("Completed migrating all Scenario Databases.")
def test_save_scenario_failure(self):
filename_field = self.save_scenario_as()
try:
filename_field.send_keys('./\\ 123.1&% AZ')
self.find('.modal.in .btn-primary').click()
time.sleep(1)
alert = self.find('.alert-danger') # this works fine in the actual program.
self.assertIn("Error", alert.text)
finally:
try:
os.remove(workspace_path('Untitled Scenario./\\ 123.1&% AZ.sqlite3'))
except:
pass
def adsm_context(request):
context = {}
if not request.is_ajax() and request.path and request.path != '/' and '/LoadingScreen/' not in request.path:
session = SmSession.objects.get()
version = session.update_available
context = {'filename': scenario_filename(), # context in either mode
'unsaved_changes': unsaved_changes(),
'url': request.path,
'active_link': '/'.join(re.split('\W+', request.path)[2:]),
'dev_version': __version__,
'update_version': version if version and version != 'False' and version != '0' else '',
'workspace_path': workspace_path(),
'db_files': (file_list(".sqlite3")),
'show_help_text': session.show_help_text
}
return context
def upload_population(request):
from ADSMSettings.models import SmSession
session = SmSession.objects.get()
if 'GET' in request.method:
json_response = {"status": session.population_upload_status, "percent": session.population_upload_percent*100}
return JsonResponse(json_response)
session.set_population_upload_status("Processing file")
if 'filename' in request.POST:
file_path = workspace_path(request.POST.get('filename'))
else:
try:
file_path = handle_file_upload(request, is_temp_file=True, overwrite_ok=True)
except FileExistsError:
return JsonResponse({"status": "failed",
"message": "Cannot import file because a file with the same name already exists in the list below."})
return parse_population(file_path, session)
def summary_csv(request):
class HttpResponseAccepted(HttpResponse):
status_code = 202
if request.method == "GET":
if SmSession.objects.get().calculating_summary_csv:
return HttpResponseAccepted()
elif not DailyControls.objects.all().count() or is_simulation_running():
return HttpResponseBadRequest()
elif not os.path.isfile(workspace_path(scenario_filename() + '/' + SUMMARY_FILE_NAME)):
return HttpResponseNotFound()
else:
return HttpResponse()
if request.method == "POST":
csv_generator = SummaryCSVGenerator()
csv_generator.start() # starts a new thread
return HttpResponseAccepted()
else:
return HttpResponseNotAllowed(permitted_methods=['GET', 'POST'])
def delete_file(request, target):
print("Deleting", target)
os.remove(workspace_path(target))
print("Done")
return HttpResponse()
def map_zip_file():
"""This is a file named after the scenario in the folder that's also named after the scenario."""
return workspace_path(scenario_filename() + '/' + scenario_filename() + " Map Output.zip")
def setUpClass(cls):
source_db = os.path.join(settings.BASE_DIR, 'ScenarioCreator', 'tests', 'population_fixtures', 'Roundtrip.sqlite3')
cls.destination_db = workspace_path('Roundtrip_test.sqlite3')
shutil.copy(source_db, cls.destination_db)
cls.scenario_directory = workspace_path('Roundtrip_test')
from django.conf import settings
import django
django.setup()
from django.core import management
from ADSMSettings.utils import db_path, workspace_path, scenario_filename
from ADSMSettings.views import new_scenario, save_scenario
from ADSMSettings.xml2sqlite import import_naadsm_xml
from ScenarioCreator.models import DirectSpread
if len(sys.argv) >= 4: # single command line invocation
print("""Usage: python3.4 ./xml2sqlite.py export_pop.xml parameters.xml debug.sqlite3 [--workspace]
Include the flag '--workspace' as the fourth argument if you want to save the scenario to the ADSM Workspace""")
shutil.copy(db_path(), workspace_path('activeSession.bak'))
shutil.copy(db_path('default'), workspace_path('settings.bak'))
popul_path = sys.argv[1]
param_path = sys.argv[2]
scenario_path = workspace_path(sys.argv[3]) if len(sys.argv) > 4 and sys.argv[4] == '--workspace' else sys.argv[3]
scenario_name = os.path.basename(scenario_path)
new_scenario()
import_naadsm_xml(popul_path, param_path, saveIterationOutputsForUnits=False)
scenario_filename(scenario_name, check_duplicates=True)
save_scenario()
count_model_entries()
shutil.move(workspace_path(scenario_name), scenario_path)
shutil.copy(workspace_path('activeSession.bak'), db_path())
def open_population(request, target):
from ADSMSettings.models import SmSession
session = SmSession.objects.get()
session.set_population_upload_status("Processing file")
return parse_population(workspace_path(target), session)
def save_scenario(request=None):
"""Save to the existing session of a new file name if target is provided
"""
if request is not None and 'filename' in request.POST:
target = request.POST['filename']
else:
target = scenario_filename()
target = strip_tags(target)
full_path = workspace_path(target) + ('.sqlite3' if not target.endswith('.sqlite3') else '')
try:
if '\\' in target or '/' in target: # this validation has to be outside of scenario_filename in order for open_test_scenario to work
raise ValueError("Slashes are not allowed: " + target)
scenario_filename(target)
print('Copying database to', target)
shutil.copy(db_path(), full_path)
unsaved_changes(False) # File is now in sync
print('Done Copying database to', full_path)
except (IOError, AssertionError, ValueError) as err:
if request is not None:
save_error = 'Failed to save filename:' + str(err)
print('Encountered an error while copying file', full_path)
return render(request, 'ScenarioName.html', {"failure_message": save_error})
if request is not None and request.is_ajax():
return render(request, 'ScenarioName.html', {"success_message": "File saved to " + target,
"filename": scenario_filename(),
'unsaved_changes': unsaved_changes()})
else:
return redirect('/setup/Scenario/1/')