def load_solution(filename):
with ArchiveReaderFactory(filename) as archive:
outf = archive.extract()
solution = None
try:
with open(outf) as f:
solution = json.load(f)
except:
solution = None
try:
solution = shelve.open(outf,
flag='r')
except:
solution = None
if solution is None:
raise RuntimeError("Unable to open ph solution file as JSON "
"or python Shelve DB format")
scenario_tree_dict = solution['scenario tree']
solution.pop('scenario tree')
return scenario_tree_dict, solution
def load_history(filename):
with ArchiveReaderFactory(filename) as archive:
outf = archive.extract()
history = None
try:
with open(outf) as f:
history = json.load(f)
except:
history = None
try:
history = shelve.open(outf, flag='r')
except:
history = None
if history is None:
raise RuntimeError("Unable to open ph history file as JSON "
"or python Shelve DB format")
scenario_tree_dict = history['scenario tree']
try:
iter_keys = history['results keys']
except KeyError:
# we are using json format (which loads the entire file
# anyway)
iter_keys = list(history.keys())
iter_keys.remove('scenario tree')
iterations = sorted(int(k) for k in iter_keys)
iterations = [str(k) for k in iterations]
return scenario_tree_dict, history, iterations
def _extract_pathspec(pathspec, default_basename, archives=None):
"""Obtain a file location from a pathspec.
Extracts a file location from the provided input
path specification by normalizing the name or by
opening an archive reader.
Args:
pathspec (str): The path specification. This can
be a standard path to a file or represent a
file contained within an archive. In the
case of an archived file, the input string
consist of two parts separated by a comma,
where the first part represents the path to
the archive and the second part represents
the relative path to a file or directory
within that archive.
default_basename (str): The default filename to
search for when the pathspec represents a
directory (or a directory within an
archive). This name must have an extension,
which is used by this function to interpret
whether the pathspec ends in a filename or a
directory name. If this argument is None, the
function will attempt to extract a directory
name instead of a file.
archives (list): A list of currently open
archive readers to check before opening a
new archive. If a new archive is opened, it will
be appended to this list.
Returns:
A tuple consisting of the normalized absolute
path to the file followed by the current list of
open archives that can be passed into this function
the next time it is called.
"""
logger.debug("expanding pathspec %s to %s" %
(pathspec, os.path.expanduser(pathspec)))
pathspec = os.path.expanduser(pathspec)
if archives is None:
archives = []
filename = None
normalized_location = None
archive = None
archive_subdir = None
unarchived_dir = None
basename = None
if not os.path.exists(pathspec):
logger.debug("pathspec does not exist, normalizing name")
(normalized_location, _, archive_subdir) = \
ArchiveReader.normalize_name(pathspec).rpartition(',')
if default_basename is not None:
extension = os.path.splitext(default_basename)[1].strip()
assert extension != ''
if archive_subdir.endswith(extension):
logger.debug("recognized extension type '%s' appears "
"after comma, treating as file" % (extension))
basename = os.path.basename(archive_subdir)
archive_subdir = os.path.dirname(archive_subdir).strip()
if archive_subdir == '':
archive_subdir = None
else:
logger.debug("pathspec exists, normalizing name")
normalized_location = \
ArchiveReader.normalize_name(pathspec)
logger.debug("normalized pathspec: (%s, %s, %s)" %
(normalized_location, archive_subdir, basename))
if ArchiveReader.isArchivedFile(normalized_location):
logger.debug("pathspec defines a recognized archive type")
for prev_archive_inputs, prev_archive, prev_unarchived_dir \
in archives:
if (normalized_location == \
prev_archive_inputs[0]) and \
((prev_archive_inputs[1] is None) or \
((archive_subdir is not None) and \
(archive_subdir.startswith(prev_archive_inputs[1]+'/')))):
logger.debug("pathspec matches previous archive")
unarchived_dir = prev_unarchived_dir
if archive_subdir is not None:
if prev_archive_inputs[1] is not None:
unarchived_dir = posixpath.join(
unarchived_dir,
os.path.relpath(archive_subdir,
start=prev_archive_inputs[1]))
else:
unarchived_dir = posixpath.join(
unarchived_dir, archive_subdir)
logger.debug("unarchived directory: %s" % (unarchived_dir))
break
else: # if no break occurs in previous for-loop
archive = ArchiveReaderFactory(normalized_location,
subdir=archive_subdir)
unarchived_dir = archive.normalize_name(
tempfile.mkdtemp(prefix='pysp_unarchived'))
archives.append(((normalized_location, archive_subdir), archive,
unarchived_dir))
logger.debug("New archive opened. Temporary archive "
"extraction directory: %s" % (unarchived_dir))
archive.extractall(path=unarchived_dir)
if basename is not None:
filename = posixpath.join(unarchived_dir, basename)
elif default_basename is not None:
filename = posixpath.join(unarchived_dir, default_basename)
else:
filename = unarchived_dir
logger.debug("extracted filename: %s" % (filename))
else:
logger.debug("pathspec defines a standard path")
if archive_subdir is not None:
unarchived_dir = posixpath.join(normalized_location,
archive_subdir)
else:
unarchived_dir = normalized_location
if not os.path.isfile(unarchived_dir):
if basename is not None:
filename = posixpath.join(unarchived_dir, basename)
elif default_basename is not None:
filename = posixpath.join(unarchived_dir, default_basename)
else:
filename = unarchived_dir
else:
filename = unarchived_dir
return filename, archives
def _extract_model_and_scenario_tree_locations(self):
model_filename = None
model_archive = None
model_archive_inputs = (None,None)
model_unarchived_dir = None
try:
# un-archive the model directory if necessary
normalized_model_location = None
model_archive_subdir = None
model_basename = "ReferenceModel.py"
if not os.path.exists(self._model_location):
normalized_model_location, _, model_archive_subdir = \
ArchiveReader.normalize_name(self._model_location).rpartition(',')
if model_archive_subdir.endswith('.py') or \
model_archive_subdir.endswith('.pyc'):
model_basename = os.path.basename(model_archive_subdir)
model_archive_subdir = os.path.dirname(model_archive_subdir)
model_archive_subdir = model_archive_subdir.strip()
if model_archive_subdir == '':
model_archive_subdir = None
else:
normalized_model_location = \
ArchiveReader.normalize_name(self._model_location)
if ArchiveReader.isArchivedFile(normalized_model_location):
model_archive = ArchiveReaderFactory(normalized_model_location,
subdir=model_archive_subdir)
self._model_archive = model_archive
model_archive_inputs = (normalized_model_location,model_archive_subdir)
model_unarchived_dir = model_archive.normalize_name(
tempfile.mkdtemp(prefix='pysp_unarchived',
dir=os.path.dirname(normalized_model_location)))
self._tmpdirs.append(model_unarchived_dir)
print("Model directory unarchiving to: %s" % model_unarchived_dir)
model_archive.extractall(path=model_unarchived_dir)
model_filename = \
posixpath.join(model_unarchived_dir, model_basename)
else:
if model_archive_subdir is not None:
model_unarchived_dir = posixpath.join(normalized_model_location,
model_archive_subdir)
else:
model_unarchived_dir = normalized_model_location
if not os.path.isfile(model_unarchived_dir):
model_filename = \
posixpath.join(model_unarchived_dir, model_basename)
else:
model_filename = model_unarchived_dir
if not os.path.exists(model_filename):
raise IOError("Model input does not exist: "+str(model_filename))
except:
print("***ERROR: Failed to locate reference "
"model file with location string: "
+str(self._model_location))
raise
self._model_filename = model_filename
self._model_directory = os.path.dirname(model_filename)
if self._scenario_tree_location is None:
return
scenario_tree_filename = None
scenario_tree_archive = None
try:
# un-archive the scenario tree directory if necessary
normalized_scenario_tree_location = None
scenario_tree_archive_subdir = None
scenario_tree_unarchived_dir = None
scenario_tree_basename = "ScenarioStructure.dat"
if not os.path.exists(self._scenario_tree_location):
normalized_scenario_tree_location, _, scenario_tree_archive_subdir = \
ArchiveReader.normalize_name(
self._scenario_tree_location).rpartition(',')
if scenario_tree_archive_subdir.endswith('.dat'):
scenario_tree_basename = \
os.path.basename(scenario_tree_archive_subdir)
scenario_tree_archive_subdir = \
os.path.dirname(scenario_tree_archive_subdir)
scenario_tree_archive_subdir = scenario_tree_archive_subdir.strip()
if scenario_tree_archive_subdir == '':
scenario_tree_archive_subdir = None
else:
normalized_scenario_tree_location = \
ArchiveReader.normalize_name(self._scenario_tree_location)
if ArchiveReader.isArchivedFile(normalized_scenario_tree_location):
if (normalized_scenario_tree_location == model_archive_inputs[0]) and \
((model_archive_inputs[1] is None) or \
((scenario_tree_archive_subdir is not None) and \
(scenario_tree_archive_subdir.startswith(model_archive_inputs[1]+'/')))):
# The scenario tree has already been extracted with the
# model archive, no need to extract again
print("Scenario tree directory found in unarchived model directory")
scenario_tree_unarchived_dir = model_unarchived_dir
if scenario_tree_archive_subdir is not None:
if model_archive_inputs[1] is not None:
scenario_tree_unarchived_dir = \
posixpath.join(
scenario_tree_unarchived_dir,
os.path.relpath(scenario_tree_archive_subdir,
start=model_archive_inputs[1]))
else:
scenario_tree_unarchived_dir = posixpath.join(
scenario_tree_unarchived_dir,
scenario_tree_archive_subdir)
else:
scenario_tree_archive = ArchiveReaderFactory(
normalized_scenario_tree_location,
subdir=scenario_tree_archive_subdir)
scenario_tree_unarchived_dir = \
scenario_tree_archive.normalize_name(
tempfile.mkdtemp(
prefix='pysp_unarchived',
dir=os.path.dirname(normalized_scenario_tree_location)))
self._tmpdirs.append(scenario_tree_unarchived_dir)
print("Scenario tree directory unarchiving to: %s"
% scenario_tree_unarchived_dir)
scenario_tree_archive.extractall(path=scenario_tree_unarchived_dir)
scenario_tree_filename = \
posixpath.join(scenario_tree_unarchived_dir,
scenario_tree_basename)
else:
if scenario_tree_archive_subdir is not None:
scenario_tree_unarchived_dir = posixpath.join(
normalized_scenario_tree_location,
scenario_tree_archive_subdir)
else:
scenario_tree_unarchived_dir = normalized_scenario_tree_location
if not os.path.isfile(scenario_tree_unarchived_dir):
scenario_tree_filename = \
posixpath.join(scenario_tree_unarchived_dir,
scenario_tree_basename)
else:
scenario_tree_filename = scenario_tree_unarchived_dir
if not os.path.exists(scenario_tree_filename):
raise IOError("Input does not exist: "+str(scenario_tree_filename))
except:
print("***ERROR: Failed to locate scenario tree structure "
"file with location string: "
+self._scenario_tree_location)
raise
self._scenario_tree_filename = scenario_tree_filename
self._scenario_tree_directory = os.path.dirname(scenario_tree_filename)
def _extract_pathspec(
pathspec,
default_basename,
archives=None):
"""Obtain a file location from a pathspec.
Extracts a file location from the provided input
path specification by normalizing the name or by
opening an archive reader.
Args:
pathspec (str): The path specification. This can
be a standard path to a file or represent a
file contained within an archive. In the
case of an archived file, the input string
consist of two parts separated by a comma,
where the first part represents the path to
the archive and the second part represents
the relative path to a file or directory
within that archive.
default_basename (str): The default filename to
search for when the pathspec represents a
directory (or a directory within an
archive). This name must have an extension,
which is used by this function to interpret
whether the pathspec ends in a filename or a
directory name. If this argument is None, the
function will attempt to extract a directory
name instead of a file.
archives (list): A list of currently open
archive readers to check before opening a
new archive. If a new archive is opened, it will
be appended to this list.
Returns:
A tuple consisting of the normalized absolute
path to the file followed by the current list of
open archives that can be passed into this function
the next time it is called.
"""
logger.debug("expanding pathspec %s to %s"
% (pathspec, os.path.expanduser(pathspec)))
pathspec = os.path.expanduser(pathspec)
if archives is None:
archives = []
filename = None
normalized_location = None
archive = None
archive_subdir = None
unarchived_dir = None
basename = None
if not os.path.exists(pathspec):
logger.debug("pathspec does not exist, normalizing name")
(normalized_location, _, archive_subdir) = \
ArchiveReader.normalize_name(pathspec).rpartition(',')
if default_basename is not None:
extension = os.path.splitext(default_basename)[1].strip()
assert extension != ''
if archive_subdir.endswith(extension):
logger.debug("recognized extension type '%s' appears "
"after comma, treating as file" % (extension))
basename = os.path.basename(archive_subdir)
archive_subdir = os.path.dirname(archive_subdir).strip()
if archive_subdir == '':
archive_subdir = None
else:
logger.debug("pathspec exists, normalizing name")
normalized_location = \
ArchiveReader.normalize_name(pathspec)
logger.debug("normalized pathspec: (%s, %s, %s)"
% (normalized_location, archive_subdir, basename))
if ArchiveReader.isArchivedFile(normalized_location):
logger.debug("pathspec defines a recognized archive type")
for prev_archive_inputs, prev_archive, prev_unarchived_dir \
in archives:
if (normalized_location == \
prev_archive_inputs[0]) and \
((prev_archive_inputs[1] is None) or \
((archive_subdir is not None) and \
(archive_subdir.startswith(prev_archive_inputs[1]+'/')))):
logger.debug("pathspec matches previous archive")
unarchived_dir = prev_unarchived_dir
if archive_subdir is not None:
if prev_archive_inputs[1] is not None:
unarchived_dir = posixpath.join(
unarchived_dir,
os.path.relpath(archive_subdir,
start=prev_archive_inputs[1]))
else:
unarchived_dir = posixpath.join(unarchived_dir,
archive_subdir)
logger.debug("unarchived directory: %s" % (unarchived_dir))
break
else: # if no break occurs in previous for-loop
archive = ArchiveReaderFactory(
normalized_location,
subdir=archive_subdir)
unarchived_dir = archive.normalize_name(
tempfile.mkdtemp(prefix='pysp_unarchived'))
archives.append(((normalized_location, archive_subdir),
archive,
unarchived_dir))
logger.debug("New archive opened. Temporary archive "
"extraction directory: %s" % (unarchived_dir))
archive.extractall(path=unarchived_dir)
if basename is not None:
filename = posixpath.join(unarchived_dir, basename)
elif default_basename is not None:
filename = posixpath.join(unarchived_dir, default_basename)
else:
filename = unarchived_dir
logger.debug("extracted filename: %s" % (filename))
else:
logger.debug("pathspec defines a standard path")
if archive_subdir is not None:
unarchived_dir = posixpath.join(normalized_location,
archive_subdir)
else:
unarchived_dir = normalized_location
if not os.path.isfile(unarchived_dir):
if basename is not None:
filename = posixpath.join(unarchived_dir, basename)
elif default_basename is not None:
filename = posixpath.join(unarchived_dir, default_basename)
else:
filename = unarchived_dir
else:
filename = unarchived_dir
return filename, archives