def import_project_from_xml(self, filename):
with self.ui_error_handler("An error has occurred.\n Your project was not imported!"):
self.model.current_project = Project.create_from_sybilla_xml(filename, parent=self.model)
self.model.current_filename = None
self.update_title()
self.view.project.present()
def run_refinement(projectf, mixture_index, options):
"""
Runs a refinement setup for
- projectf: project data
- mixture_index: what mixture in the project to use
- options: refinement options
"""
if projectf is not None:
from pyxrd.data import settings
settings.CACHE = None
settings.apply_runtime_settings()
from pyxrd.generic import pool
pool.get_pool()
from pyxrd.project.models import Project
type(Project).object_pool.clear()
project = Project.load_object(None, data=projectf)
del projectf
import gc
gc.collect()
mixture = project.mixtures[mixture_index]
mixture.update_refinement_treestore()
mixture.refiner.setup_context(store=False) # we already have a dumped project
context = mixture.refiner.context
context.options = options
mixture.refiner.refine(stop=pool.pool_stop)
return list(context.best_solution), context.best_residual, (context.record_header, context.records) #@UndefinedVariable
def run(args):
if args and args.filename != "":
project = Project.load_object(args.filename)
default_atoms = []
AtomType.get_from_csv(
settings.DATA_REG.get_file_path("ATOM_SCAT_FACTORS"),
default_atoms.append
)
for atom in project.atom_types.iter_objects():
default_atom = None
for def_atom in default_atoms:
if atom.name == def_atom.name:
default_atom = def_atom
assert default_atom is not None
atom.charge = default_atom.charge
atom.debye = default_atom.debye
project.save_object(args.filename + ".def")
del project
print "Finished!"
def open_project(self, filename):
# Try to load the project:
with self.ui_error_handler("An error has occurred.\n Your project was not loaded!"):
self.model.current_project = Project.load_object(filename, parent=self.model)
# Set the current filename property and update the title
self.model.current_filename = filename
self.update_title()
def phaseworker(in_queue, save_queue, stop):
"""
Parses Phase descriptions into actual objects and passes them
to the save_queue.
'stop' should be a threading.Event() that should be toggled
once all elements have been Queued.
This way, the worker will only stop once the Queue is really empty,
and not when it's processing faster than the Queue can be filled.
"""
while True:
try:
phases_path, phase_descr = in_queue.get(timeout=0.5)
project = Project()
phase_lookup = {}
component_lookup = {}
for phase_kwargs, code, comp_props in phase_descr:
# create phase:
G = len(code) / code_length
based_on = None
if "based_on" in phase_kwargs:
based_on = phase_lookup.get(phase_kwargs.pop("based_on"), None)
phase = Phase(G=G, parent=project, **phase_kwargs)
phase.based_on = based_on
phase_lookup[phase.name] = phase
# derive upper and lower limits for the codes using code lengths:
limits = list(zip(
list(range(0, len(code), code_length)),
list(range(code_length, len(code) + 1, code_length))
))
# create components:
phase.components[:] = []
for ll, ul in limits:
part = code[ll: ul]
for component in Component.load_components(aliases[part] % (settings.DATA_REG.get_directory_path("DEFAULT_COMPONENTS") + "/"), parent=phase):
component.resolve_json_references()
phase.components.append(component)
props = comp_props.get(part, {})
for prop, value in props.items():
if prop == 'linked_with':
value = component_lookup[value]
setattr(component, prop, value)
component_lookup[part] = component
# put phases on the save queue:
phases_path = phases_path % (settings.DATA_REG.get_directory_path("DEFAULT_PHASES") + "/")
save_queue.put((phases_path, list(phase_lookup.values())))
# Flag this as finished
in_queue.task_done()
except queue.Empty:
if not stop.is_set():
continue
else:
return
def new_project(self, *args, **kwargs):
# Create a new project
self.model.current_project = Project(parent=self.model)
# Set the current filename property and update the title
self.update_title()
# Show the edit project dialog
self.view.project.present()
def _run_gui(project=None):
# Display a splash screen showing the loading status...
from pkg_resources import resource_filename # @UnresolvedImport
from pyxrd.application.splash import SplashScreen
from pyxrd import __version__
filename = resource_filename(__name__, "application/icons/pyxrd.png")
splash = SplashScreen(filename, __version__)
# Run GUI:
splash.set_message("Loading GUI ...")
# Now we can load these:
from pyxrd.data import settings
from pyxrd.project.models import Project
from pyxrd.application.models import AppModel
from pyxrd.application.views import AppView
from pyxrd.application.controllers import AppController
from pyxrd.generic.gtk_tools.gtkexcepthook import plugin_gtk_exception_hook
filename = settings.ARGS.filename #@UndefinedVariable
# Check if a filename was passed, if so try to load it
if filename != "":
try:
logging.info("Opening project: %s" % filename)
project = Project.load_object(filename)
except IOError:
logging.info("Could not load project file %s: IOError" % filename)
# FIXME the user should be informed of this in a dialog...
# Disable unity overlay scrollbars as they cause bugs with modal windows
os.environ['LIBOVERLAY_SCROLLBAR'] = '0'
os.environ['UBUNTU_MENUPROXY'] = ""
if not settings.DEBUG:
warnings.filterwarnings(action='ignore', category=Warning)
# Close splash screen
if splash: splash.close()
# Nice GUI error handler:
gtk_exception_hook = plugin_gtk_exception_hook()
# setup MVC:
m = AppModel(project=project)
v = AppView()
AppController(m, v, gtk_exception_hook=gtk_exception_hook)
# Free this before continuing
del splash
# lets get this show on the road:
gtk.main()
def run(args):
# batch csv file generator for a file containin a number of projects
if args and args.filename != "":
project = Project.load_object(args.filename)
for i, mixture in enumerate(project.mixtures):
mixture.update_refinement_treestore()
refs = []
props = ("d001", "F1", "F2", "F3", "F4", "W1", "P11_or_P22",)
for node in mixture.refinables.iter_children():
ref = node.object
if ref.refinable and ref.prop in props:
val = ref.value
delta = 0.1
if val > 1.0:
delta = log(val)
ref.value_min = val - delta
ref.value_max = val + delta
refs.append(ref)
combs = list(combinations(refs, 2))
fractions = np.zeros(shape=(len(combs), len(mixture.fractions)))
rps = np.zeros(shape=(len(combs),))
print len(combs) * 9, " calculations to be made!"
for k, (ref1, ref2) in enumerate(combs):
print "\rProgress: %00.f%%" % float(100.0 * k / len(combs)),
for i in range(3):
for j in range(3):
ref1.value = ref1.value_min + float(i) * (ref1.value_max - ref1.value_min) / 2.0
ref2.value = ref2.value_min + float(j) * (ref2.value_max - ref2.value_min) / 2.0
rp = mixture.optimize()
if rp <= 30.0:
inv_rp = 30.0 - rp
fractions[k, :] = np.array(mixture.fractions, dtype=float)
rps[k] = inv_rp
# print "\rProgress: %00f%% - iter %d of 25" % (float(100.0 * k / len(combs)), i*5+(j+1))
print "Phases:"
print mixture.phases
print "Unweighted average:"
print np.average(fractions.transpose(), axis=1, weights=rps)
print np.std(fractions.transpose() * rps, axis=1) / np.sum(rps)
print "Weighted average:"
print np.average(fractions.transpose(), axis=1, weights=rps)
print np.std(fractions.transpose(), axis=1)
print "Average Rp value:"
print np.average(30.0 - rps), "+/-", np.std(30.0 - rps)
del project
print "Finished!"
def setup_project(projectf):
# Only import these at this point, the cause a considerable increase in
# memory usage, so if no projectf was passed to improve_solutions, this
# module does not use more then it needs.
from pyxrd.data import settings
settings.apply_runtime_settings()
from pyxrd.project.models import Project
type(Project).object_pool.clear()
f = StringIO()
f.write(projectf)
project = Project.load_object(f)
f.close()
return project
def run(args):
# generates a project file containing the phases as described by the Sybilla XML output:
if args and args.filename != "":
# Import:
project = Project.create_from_sybilla_xml(args.filename)
# Save this right away:
project_filename = "%s/%s" % (os.path.dirname(args.filename), os.path.basename(args.filename).replace(".xml", ".pyxrd", 1))
project.save_object(project_filename)
# Relaunch processs
args = [sys.argv[0], project_filename, ]
args.insert(0, sys.executable)
if sys.platform == 'win32':
args = ['"%s"' % arg for arg in args]
os.execv(sys.executable, args)
sys.exit(0)
def run(args):
# batch csv file generator for a file containin a number of projects
if args and args.filename != "":
projects = []
with open(args.filename, 'r') as f:
for project_file in f:
project_file = project_file.rstrip()
print "Parsing: %s" % project_file
project = Project.load_object(project_file)
for i, mixture in enumerate(project.mixtures):
np.savetxt(
"%s/%s" % (os.path.dirname(project_file), os.path.basename(project_file).replace(".pyxrd", "-%d.csv" % i, 1)),
get_result_description(mixture), fmt='%s', delimiter=';'
)
del project
print "Finished!"
def run(args):
# generates a project file containing the phases as described by the Sybilla XML output:
if args and args.filename != "":
# Import:
project = Project.create_from_sybilla_xml(args.filename)
# Save this right away:
project_filename = "%s/%s" % (os.path.dirname(
args.filename), os.path.basename(args.filename).replace(
".xml", ".pyxrd", 1))
from pyxrd.file_parsers.json_parser import JSONParser
JSONParser.write(project, project_filename, zipped=True)
# Relaunch processs
args = [
sys.argv[0],
project_filename,
]
args.insert(0, sys.executable)
if sys.platform == 'win32':
args = ['"%s"' % arg for arg in args]
os.execv(sys.executable, args)
sys.exit(0)
def run(args):
"""
This is a simple script that will open a PyXRD project file,
will run a refinement for a certain mixture, and store the results
in an overview file and the best solution as a new project file.
The refinement setup is left unchanged, so be sure you have correctly
defined parameter ranges and chosen a good refinement strategy (CMA-ES
is recommended).
To use this script, launch it using PyXRD's core.py launcher script as:
python core.py -s pyxrd/scripts/refinement_generator.py "$FILENAME###$I###$J"
in which:
- $FILENAME can be replaced with the absolute path to the actual
project filename
- $I is the index of the mixture to refine and
- $J is the 'trial' number, which is added to the record file and to the
best solution project file.
- leave the three # (hashes) where they are, they are used as separators
You can use this script (e.g. on high-performance computing clusters) to
run several iterations of the same project. Reasaons why you would want
to do this are for benchmarking, checking solution reliability, ...
Just change the trial number from e.g. 0 to 49 to have 50 iterations.
"""
## TODO:
## - use a uniform distribution of starting solutions:
## xs = np.random.uniform(size=50)
## ys = np.random.uniform(size=50)
## zs = np.random.uniform(size=50)
##
## When the jobs are submitted, load the project and mixture once,
## create the # of staring solutions and store them in a file (using np IO)
## Then here we can load them and pick the one we need.
##
if args and args.filename != "":
logging.info("Proccessing args...")
project_file, k, mixture_index = tuple(args.filename.split("###", 2))
base_path = os.path.dirname(args.filename)
start_solutions_fname = os.path.join(
base_path,
"start_solutions %s mixture %s" % (os.path.basename(project_file), mixture_index)
)
stop_event = multiprocessing.Event()
logging.info("Loading project file...")
project = Project.load_object(project_file)
logging.info(" ".join(["Running Project", os.path.basename(project_file), "Trial", k]))
for i, mixture in enumerate(project.mixtures):
if i == int(mixture_index):
if B_DO_PROFILING:
pr = cProfile.Profile()
pr.enable()
try:
with mixture.data_changed.hold():
mixture.update_refinement_treestore()
mixture.refiner.setup_context(store=True)
if int(k) == 0: #First run, create solutions & store for later use:
start_solutions = mixture.refiner.context.get_uniform_solutions(50)
np.savetxt(start_solutions_fname, start_solutions)
else:
start_solutions = np.loadtxt(start_solutions_fname)
mixture.refiner.context.set_initial_solution(start_solutions[k, ...])
mixture.optimizer.optimize()
mixture.refiner.refine(stop=stop_event)
except:
raise
finally:
if B_DO_PROFILING:
pr.disable()
with open("pyxrd_stats", "w+") as f:
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=f).sort_stats(sortby)
ps.print_stats()
context = mixture.refiner.context
recordf = os.path.basename(project_file).replace(".pyxrd", "")
recordf = base_path + "/" + "record#" + str(k) + " " + recordf + " " + mixture.name
with codecs.open(recordf, 'w', 'utf-8') as f:
f.write("################################################################################\n")
f.write(recordf + "\n")
f.write("Mixture " + str(i) + " and trial " + str(k) + "\n")
f.write("Property name, initial, best, min, max" + "\n")
for j, ref_prop in enumerate(context.ref_props):
line = ", ".join([
ref_prop.get_descriptor(),
str(context.initial_solution[j]),
str(context.best_solution[j]),
str(ref_prop.value_min),
str(ref_prop.value_max),
])
f.write(line + "\n")
f.write("################################################################################\n")
def write_records(f, record_header, records):
f.write(", ".join(record_header) + "\n")
for record in records:
f.write(", ".join(map(lambda f: "%.7f" % f, record)) + "\n")
f.write("################################################################################\n")
if context.record_header is not None:
write_records(f, context.record_header, context.records)
if hasattr(context, "pcma_records"):
for record_header, records in context.pcma_records:
write_records(f, record_header, records)
# Apply found solution and save:
context.apply_best_solution()
mixture.optimizer.optimize()
project_file_output = base_path + "/" + os.path.basename(project_file).replace(".pyxrd", "") + " - mixture %s - trial %s.pyxrd" % (str(i), str(k))
project.save_object(file=project_file_output)
pass # end
def setUp(self):
mock_settings()
self.project = Project(name="Test Project")
def run(args):
project = Project(name="Test")
project.name = "Test123"
pass
def setUp(self):
mock_settings()
self.project = Project(name="TestProject")
self.mixture = Mixture(name="TestMixture", parent=self.project)
def create_project_from_sybilla_xml(filename, **kwargs):
"""
Creates a new project structure from a Sybilla XML file.
Some information (e.g. the actual XRD pattern) is not present and will
still need to be imported manually.
"""
tree = ET.parse(filename)
root = tree.getroot()
basename = os.path.basename(filename)
# Create the project:
if "name" in kwargs: kwargs.pop("name")
if "layout_mode" in kwargs: kwargs.pop("layout_mode")
project = Project(name=basename, layout_mode="FULL", **kwargs)
# Add a specimen:
specimen = Specimen(name=basename, parent=project)
project.specimens.append(specimen)
# Add a mixture:
mixture = Mixture(name=basename, auto_run=False, parent=project)
mixture.add_specimen_slot(specimen, 1.0, 0.0)
project.mixtures.append(mixture)
with project.data_changed.ignore():
with mixture.data_changed.ignore():
for child in root:
if child.tag == "basic_params":
# Goniometer parameters:
step_size = safe_float(child.attrib['step_size'])
wavelength = safe_float(child.attrib['lambda']) / 10.0
steps = int(1 +
(specimen.goniometer.max_2theta -
specimen.goniometer.min_2theta) / step_size)
specimen.goniometer.min_2theta = safe_float(
child.attrib['min2theta'])
specimen.goniometer.max_2theta = safe_float(
child.attrib['max2theta'])
specimen.goniometer.steps = steps
specimen.goniometer.wavelength = wavelength
elif child.tag == "diffractometer":
# Some more goniometer parameters, and specimen parameters:
specimen.goniometer.radius = safe_float(
child.attrib['gonio_radius'])
specimen.goniometer.divergence = safe_float(
child.attrib['diverg_slit'])
specimen.goniometer.soller1 = safe_float(
child.attrib['Soller1'])
specimen.goniometer.soller2 = safe_float(
child.attrib['Soller2'])
specimen.sample_length = safe_float(
child.attrib['sample_length'])
elif child.tag == "content":
# Content tag contains 'Mixture' data
for xmlPhaseContent in child:
name = xmlPhaseContent.attrib['name']
fraction = safe_float(
xmlPhaseContent.attrib['content']) / 100.
mixture.add_phase_slot(name, fraction)
elif child.tag == "mixture":
# Mixture tag corresponds with the phases in the project level,
# not an actual Mixture object:
for xmlPhase in child:
name = xmlPhase.attrib['name']
sigma = xmlPhase.attrib['sigma_star']
csds = safe_float(
xmlPhase.find('distribution').attrib['Tmean'])
G = 1
R = 0
W = [
1.0,
]
if xmlPhase.attrib['type'] != 'mono':
prob = xmlPhase.find('probability')
G = int(prob.attrib['no_of_comp'])
R = int(prob.attrib['R'])
# create phase and add to project:
phase = Phase(name=name,
sigma_star=sigma,
G=G,
R=R,
parent=project)
phase.CSDS_distribution.average = csds
project.phases.append(phase)
# set probability:
if R == 0 and G != 1:
xmlW = prob.find('W')
W = np.array([
float(
int(
safe_float(xmlW.attrib[
string.ascii_lowercase[i]]) *
1000.)) / 1000. for i in range(G)
])
for i in range(G - 1):
setattr(phase.probabilities, "F%d" % (i + 1),
W[i] / np.sum(W[i:]))
if R == 1 and G == 2:
pass # TODO
# ... TODO other probs
# parse components:
for i, layer in enumerate(
xmlPhase.findall("./layer_and_edge/layer")):
component = phase.components[i]
component.name = layer.attrib['name']
component.d001 = safe_float(
layer.attrib['d_spacing']) / 10.0
component.default_c = safe_float(
layer.attrib['d_spacing']) / 10.0
component.delta_c = safe_float(
layer.attrib['d_spacing_delta']) / 10.0
component.ucp_b.value = 0.9
component.ucp_a.factor = 0.57735
component.ucp_a.prop = (component, 'cell_b')
component.ucp_a.enabled = True
atom_type_map = {
# "NH4": "FIXME"
"K": "K1+",
"O": "O1-",
"Si": "Si2+",
"OH": "OH1-",
"Fe": "Fe1.5+",
"Al": "Al1.5+",
"Mg": "Mg1+",
"H2O": "H2O",
"Gly": "Glycol",
"Ca": "Ca2+",
"Na": "Na1+",
}
# add atoms:
fe_atom = None
encountered_oxygen = False
for atom in layer.findall("atom"):
atom_type_name = atom_type_map.get(
atom.attrib['type'], None)
if atom_type_name:
if atom_type_name == "O1-":
# From this point we're dealing with layer atoms
encountered_oxygen = True
atom = Atom(
name=atom.attrib['type'],
default_z=safe_float(
atom.attrib['position']) / 10.0,
pn=safe_float(atom.attrib['content']),
atom_type_name=atom_type_name,
parent=component)
if encountered_oxygen:
component.layer_atoms.append(atom)
else:
component.interlayer_atoms.append(atom)
atom.resolve_json_references()
# Assume this is the octahedral iron...
if encountered_oxygen and atom_type_name == "Fe1.5+":
fe_atom = atom
# Set the atom relation
if fe_atom is not None:
component.ucp_b.constant = 0.9
component.ucp_b.factor = 0.0043
component.ucp_b.prop = (fe_atom, 'pn')
component.ucp_b.enabled = True
pass # end of if
pass # end of for
# Map phases onto mixture names:
for phase in project.phases:
for slot, phase_name in enumerate(mixture.phases):
if phase.name == phase_name:
mixture.set_phase(0, slot, phase)
return project
def test_parent(self):
parent_project = Project(name="Parent2")
self.mixture.parent = parent_project
self.assertEqual(self.mixture.parent, parent_project)
def run(args):
project = Project(name="Test")
project.name = "Test123"
pass
def setUp(self):
self.project = Project.load_object(resource_filename("test.test_mixture", "test refinement.pyxrd"))
self.mixture = self.project.mixtures[0]