def test_decoding(self):
child_encoded = """
{
"type": "DummyStorable",
"properties": {
"name": "Child Dummy",
"data": [
0,
1,
2,
3,
4
],
"my_daddy": {
"type": "DummyStorable",
"properties": {
"name": "Daddy Dummy",
"data": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ],
"my_daddy": null
}
}
}
}"""
f = StringIO(child_encoded)
decoded_child = JSONParser.parse(f)
self.assertNotEqual(decoded_child, None)
self.assertNotEqual(decoded_child.my_daddy, None)
def test_decoding(self):
child_encoded = """
{
"type": "DummyStorable",
"properties": {
"name": "Child Dummy",
"data": [
0,
1,
2,
3,
4
],
"my_daddy": {
"type": "DummyStorable",
"properties": {
"name": "Daddy Dummy",
"data": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ],
"my_daddy": null
}
}
}
}"""
f = StringIO(child_encoded)
decoded_child = JSONParser.parse(f)
self.assertNotEqual(decoded_child, None)
self.assertNotEqual(decoded_child.my_daddy, None)
def load_components(cls, filename, parent=None):
"""
Returns multiple components loaded from a single file.
"""
# Before import, we change all the UUID's
# This way we're sure that we're not going to import objects
# with duplicate UUID's!
type(cls).object_pool.change_all_uuids()
if zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, 'r') as zfile:
for uuid in zfile.namelist():
obj = JSONParser.parse(zfile.open(uuid))
obj.parent = parent
yield obj
else:
obj = JSONParser.parse(filename)
obj.parent = parent
yield obj
def load_components(cls, filename, parent=None):
"""
Returns multiple components loaded from a single file.
"""
# Before import, we change all the UUID's
# This way we're sure that we're not going to import objects
# with duplicate UUID's!
type(cls).object_pool.change_all_uuids()
if zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, 'r') as zfile:
for uuid in zfile.namelist():
obj = JSONParser.parse(zfile.open(uuid))
obj.parent = parent
yield obj
else:
obj = JSONParser.parse(filename)
obj.parent = parent
yield obj
def setup_project(projectf):
from pyxrd.file_parsers.json_parser import JSONParser
from pyxrd.project.models import Project
type(Project).object_pool.clear()
f = StringIO(projectf)
project = JSONParser.parse(f)
f.close()
return project
def setup_project(projectf):
from pyxrd.file_parsers.json_parser import JSONParser
from pyxrd.project.models import Project
type(Project).object_pool.clear()
f = StringIO(projectf)
project = JSONParser.parse(f)
f.close()
return project
def reset_from_file(self, gonfile):
"""
Loads & sets the parameters from the goniometer JSON file
specified by `gonfile`, can be a filename or a file-like object.
"""
new_gonio = JSONParser.parse(gonfile)
with self.data_changed.hold():
for prop in self.Meta.all_properties:
if prop.storable:
if prop.name == "wavelength_distribution":
self.wavelength_distribution.clear()
self.wavelength_distribution.set_data(
*new_gonio.wavelength_distribution.get_xy_data())
elif prop.name != "uuid":
setattr(self, prop.name, getattr(new_gonio, prop.name))
def reset_from_file(self, gonfile):
"""
Loads & sets the parameters from the goniometer JSON file
specified by `gonfile`, can be a filename or a file-like object.
"""
new_gonio = JSONParser.parse(gonfile)
with self.data_changed.hold():
for prop in self.Meta.all_properties:
if prop.persistent:
if prop.label == "wavelength_distribution":
self.wavelength_distribution.clear()
self.wavelength_distribution.set_data(
*new_gonio.wavelength_distribution.get_xy_data())
elif prop.label != "uuid":
setattr(self, prop.label, getattr(new_gonio, prop.label))
def run(args):
"""
This is a simple script that will open a PyXRD project file,
will run a refinement for a certain mixture, and store the results
in an overview file and the best solution as a new project file.
The refinement setup is left unchanged, so be sure you have correctly
defined parameter ranges and chosen a good refinement strategy (CMA-ES
is recommended).
To use this script, launch it using PyXRD's core.py launcher script as:
python core.py -s pyxrd/scripts/refinement_generator.py "$FILENAME###$I###$J"
in which:
- $FILENAME can be replaced with the absolute path to the actual
project filename
- $I is the index of the mixture to refine and
- $J is the 'trial' number, which is added to the record file and to the
best solution project file.
- leave the three # (hashes) where they are, they are used as separators
You can use this script (e.g. on high-performance computing clusters) to
run several iterations of the same project. Reasaons why you would want
to do this are for benchmarking, checking solution reliability, ...
Just change the trial number from e.g. 0 to 49 to have 50 iterations.
"""
## TODO:
## - use a uniform distribution of starting solutions:
## xs = np.random.uniform(size=50)
## ys = np.random.uniform(size=50)
## zs = np.random.uniform(size=50)
##
## When the jobs are submitted, load the project and mixture once,
## create the # of staring solutions and store them in a file (using np IO)
## Then here we can load them and pick the one we need.
##
if args and args.filename != "":
logging.info("Proccessing args...")
project_file, k, mixture_index = tuple(args.filename.split("###", 2))
base_path = os.path.dirname(args.filename)
start_solutions_fname = os.path.join(
base_path,
"start_solutions %s mixture %s" % (os.path.basename(project_file), mixture_index)
)
stop_event = multiprocessing.Event()
logging.info("Loading project file...")
from pyxrd.file_parsers.json_parser import JSONParser
project = JSONParser.parse(project_file)
logging.info(" ".join(["Running Project", os.path.basename(project_file), "Trial", k]))
for i, mixture in enumerate(project.mixtures):
if i == int(mixture_index):
if B_DO_PROFILING:
pr = cProfile.Profile()
pr.enable()
try:
with mixture.data_changed.hold():
mixture.refinement.update_refinement_treestore()
refiner = mixture.refinement.get_refiner()
if int(k) == 0: #First run, create solutions & store for later use:
start_solutions = get_uniform_solutions(refiner, 50)
np.savetxt(start_solutions_fname, start_solutions)
else:
start_solutions = np.loadtxt(start_solutions_fname)
refiner.update(start_solutions[k, ...], iteration=-1)
mixture.optimizer.optimize()
refiner.refine(stop_event)
except:
raise
finally:
if B_DO_PROFILING:
pr.disable()
with open("pyxrd_stats", "w+") as f:
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=f).sort_stats(sortby)
ps.print_stats()
recordf = os.path.basename(project_file).replace(".pyxrd", "")
recordf = base_path + "/" + "record#" + str(k) + " " + recordf + " " + mixture.name
with codecs.open(recordf, 'w', 'utf-8') as f:
f.write("################################################################################\n")
f.write(recordf + "\n")
f.write("Mixture " + str(i) + " and trial " + str(k) + "\n")
f.write("Property name, initial, best, min, max" + "\n")
for j, ref_prop in enumerate(refiner.refinable_properties):
line = ", ".join([
ref_prop.get_descriptor(),
str(refiner.history.initial_solution[j]),
str(refiner.history.best_solution[j]),
str(ref_prop.value_min),
str(ref_prop.value_max),
])
f.write(line + "\n")
f.write("################################################################################\n")
def write_records(f, record_header, records):
f.write(", ".join(record_header) + "\n")
for record in records:
f.write(", ".join(map(lambda f: "%.7f" % f, record)) + "\n")
f.write("################################################################################\n")
# Apply found solution and save:
refiner.apply_best_solution()
mixture.optimizer.optimize()
project_file_output = base_path + "/" + os.path.basename(project_file).replace(".pyxrd", "") + " - mixture %s - trial %s.pyxrd" % (str(i), str(k))
JSONParser.write(project, project_file_output, zipped=True)
pass # end
def setUp(self):
self.project = JSONParser.parse(
resource_filename("test.test_mixture", "test refinement.pyxrd"))
self.mixture = self.project.mixtures[0]
def run(args):
"""
This is a simple script that will open a PyXRD project file,
will run a refinement for a certain mixture, and store the results
in an overview file and the best solution as a new project file.
The refinement setup is left unchanged, so be sure you have correctly
defined parameter ranges and chosen a good refinement strategy (CMA-ES
is recommended).
To use this script, launch it using PyXRD's core.py launcher script as:
python core.py -s pyxrd/scripts/refinement_generator.py "$FILENAME###$I###$J"
in which:
- $FILENAME can be replaced with the absolute path to the actual
project filename
- $I is the index of the mixture to refine and
- $J is the 'trial' number, which is added to the record file and to the
best solution project file.
- leave the three # (hashes) where they are, they are used as separators
You can use this script (e.g. on high-performance computing clusters) to
run several iterations of the same project. Reasaons why you would want
to do this are for benchmarking, checking solution reliability, ...
Just change the trial number from e.g. 0 to 49 to have 50 iterations.
"""
## TODO:
## - use a uniform distribution of starting solutions:
## xs = np.random.uniform(size=50)
## ys = np.random.uniform(size=50)
## zs = np.random.uniform(size=50)
##
## When the jobs are submitted, load the project and mixture once,
## create the # of staring solutions and store them in a file (using np IO)
## Then here we can load them and pick the one we need.
##
if args and args.filename != "":
logging.info("Proccessing args...")
project_file, k, mixture_index = tuple(args.filename.split("###", 2))
base_path = os.path.dirname(args.filename)
start_solutions_fname = os.path.join(
base_path,
"start_solutions %s mixture %s" % (os.path.basename(project_file), mixture_index)
)
stop_event = multiprocessing.Event()
logging.info("Loading project file...")
from pyxrd.file_parsers.json_parser import JSONParser
project = JSONParser.parse(project_file)
logging.info(" ".join(["Running Project", os.path.basename(project_file), "Trial", k]))
for i, mixture in enumerate(project.mixtures):
if i == int(mixture_index):
if B_DO_PROFILING:
pr = cProfile.Profile()
pr.enable()
try:
with mixture.data_changed.hold():
mixture.refinement.update_refinement_treestore()
refiner = mixture.refinement.get_refiner()
if int(k) == 0: #First run, create solutions & store for later use:
start_solutions = get_uniform_solutions(refiner, 50)
np.savetxt(start_solutions_fname, start_solutions)
else:
start_solutions = np.loadtxt(start_solutions_fname)
refiner.update(start_solutions[k, ...], iteration=-1)
mixture.optimizer.optimize()
refiner.refine(stop_event)
except:
raise
finally:
if B_DO_PROFILING:
pr.disable()
with open("pyxrd_stats", "w+") as f:
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=f).sort_stats(sortby)
ps.print_stats()
recordf = os.path.basename(project_file).replace(".pyxrd", "")
recordf = base_path + "/" + "record#" + str(k) + " " + recordf + " " + mixture.name
with codecs.open(recordf, 'w', 'utf-8') as f:
f.write("################################################################################\n")
f.write(recordf + "\n")
f.write("Mixture " + str(i) + " and trial " + str(k) + "\n")
f.write("Property name, initial, best, min, max" + "\n")
for j, ref_prop in enumerate(refiner.refinable_properties):
line = ", ".join([
ref_prop.get_descriptor(),
str(refiner.history.initial_solution[j]),
str(refiner.history.best_solution[j]),
str(ref_prop.value_min),
str(ref_prop.value_max),
])
f.write(line + "\n")
f.write("################################################################################\n")
def write_records(f, record_header, records):
f.write(", ".join(record_header) + "\n")
for record in records:
f.write(", ".join(["%.7f" % f for f in record]) + "\n")
f.write("################################################################################\n")
# Apply found solution and save:
refiner.apply_best_solution()
mixture.optimizer.optimize()
project_file_output = base_path + "/" + os.path.basename(project_file).replace(".pyxrd", "") + " - mixture %s - trial %s.pyxrd" % (str(i), str(k))
JSONParser.write(project, project_file_output, zipped=True)
pass # end
def _run_gui(project=None):
# Display a splash screen showing the loading status...
from pkg_resources import resource_filename # @UnresolvedImport
from pyxrd.application.splash import SplashScreen
from pyxrd import __version__
filename = resource_filename(__name__, "application/icons/pyxrd.png")
splash = SplashScreen(filename, __version__)
# Run GUI:
splash.set_message("Checking for updates ...")
update_available, nv, cv = check_for_updates()
splash.set_message("Loading GUI ...")
# Now we can load these:
from pyxrd.data import settings
from pyxrd.file_parsers.json_parser import JSONParser
from pyxrd.application.models import AppModel
from pyxrd.application.views import AppView
from pyxrd.application.controllers import AppController
# TODO move this to mvc:
from pyxrd.generic.gtk_tools.gtkexcepthook import plugin_gtk_exception_hook
filename = settings.ARGS.filename #@UndefinedVariable
# Check if a filename was passed, if so try to load it
if filename != "":
try:
logging.info("Opening project: %s" % filename)
project = JSONParser.parse(filename)
except IOError:
logging.info("Could not load project file %s: IOError" % filename)
# FIXME the user should be informed of this in a dialog...
# Disable unity overlay scrollbars as they cause bugs with modal windows
os.environ['LIBOVERLAY_SCROLLBAR'] = '0'
os.environ['UBUNTU_MENUPROXY'] = ""
if not settings.DEBUG:
warnings.filterwarnings(action='ignore', category=Warning)
# Close splash screen
if splash:
splash.close()
# Nice GUI error handler:
gtk_exception_hook = plugin_gtk_exception_hook()
# setup MVC:
m = AppModel(project=project)
v = AppView()
AppController(m, v, gtk_exception_hook=gtk_exception_hook)
# Free this before continuing
del splash
if update_available:
from mvc.adapters.gtk_support.dialogs.dialog_factory import DialogFactory
DialogFactory.get_information_dialog(
"An update is available (%s) - consider upgrading!" % nv,
False, v.get_toplevel(), title = "Update available"
).run()
else:
print("PyXRD is up to date (current = %s)" % cv)
# lets get this show on the road:
start_event_loop()
def _run_gui(project=None):
# Display a splash screen showing the loading status...
from pkg_resources import resource_filename # @UnresolvedImport
from pyxrd.application.splash import SplashScreen
from pyxrd import __version__
filename = resource_filename(__name__, "application/icons/pyxrd.png")
splash = SplashScreen(filename, __version__)
# Run GUI:
splash.set_message("Checking for updates ...")
update_available, nv, cv = check_for_updates()
splash.set_message("Loading GUI ...")
# Now we can load these:
from pyxrd.data import settings
from pyxrd.file_parsers.json_parser import JSONParser
from pyxrd.application.models import AppModel
from pyxrd.application.views import AppView
from pyxrd.application.controllers import AppController
# TODO move this to mvc:
from pyxrd.generic.gtk_tools.gtkexcepthook import plugin_gtk_exception_hook
filename = settings.ARGS.filename #@UndefinedVariable
# Check if a filename was passed, if so try to load it
if filename != "":
try:
logging.info("Opening project: %s" % filename)
project = JSONParser.parse(filename)
except IOError:
logging.info("Could not load project file %s: IOError" % filename)
# FIXME the user should be informed of this in a dialog...
# Disable unity overlay scrollbars as they cause bugs with modal windows
os.environ['LIBOVERLAY_SCROLLBAR'] = '0'
os.environ['UBUNTU_MENUPROXY'] = ""
if not settings.DEBUG:
warnings.filterwarnings(action='ignore', category=Warning)
# Close splash screen
if splash:
splash.close()
# Nice GUI error handler:
gtk_exception_hook = plugin_gtk_exception_hook()
# setup MVC:
m = AppModel(project=project)
v = AppView()
AppController(m, v, gtk_exception_hook=gtk_exception_hook)
# Free this before continuing
del splash
if update_available:
from mvc.adapters.gtk_support.dialogs.dialog_factory import DialogFactory
DialogFactory.get_information_dialog(
"An update is available (%s) - consider upgrading!" % nv,
False,
v.get_toplevel(),
title="Update available").run()
else:
print("PyXRD is up to date (current = %s)" % cv)
# lets get this show on the road:
start_event_loop()
def setUp(self):
self.project = JSONParser.parse(resource_filename("test.test_mixture", "test refinement.pyxrd"))
self.mixture = self.project.mixtures[0]