Example #1
0
    def import_project_from_xml(self, filename):
        with self.ui_error_handler("An error has occurred.\n Your project was not imported!"):
            self.model.current_project = Project.create_from_sybilla_xml(filename, parent=self.model)

        self.model.current_filename = None
        self.update_title()
        self.view.project.present()
Example #2
0
def run_refinement(projectf, mixture_index, options):
    """
        Runs a refinement setup for 
            - projectf: project data
            - mixture_index: what mixture in the project to use
            - options: refinement options
    """
    if projectf is not None:
        from pyxrd.data import settings
        settings.CACHE = None
        settings.apply_runtime_settings()

        from pyxrd.generic import pool
        pool.get_pool()

        from pyxrd.project.models import Project

        type(Project).object_pool.clear()
        project = Project.load_object(None, data=projectf)
        del projectf
        import gc
        gc.collect()

        mixture = project.mixtures[mixture_index]
        mixture.update_refinement_treestore()
        mixture.refiner.setup_context(store=False) # we already have a dumped project
        context = mixture.refiner.context
        context.options = options

        mixture.refiner.refine(stop=pool.pool_stop)

        return list(context.best_solution), context.best_residual, (context.record_header, context.records) #@UndefinedVariable
def run(args):
    if args and args.filename != "":
        project = Project.load_object(args.filename)

        default_atoms = []

        AtomType.get_from_csv(
            settings.DATA_REG.get_file_path("ATOM_SCAT_FACTORS"),
            default_atoms.append
        )

        for atom in project.atom_types.iter_objects():

            default_atom = None
            for def_atom in default_atoms:
                if atom.name == def_atom.name:
                    default_atom = def_atom
            assert default_atom is not None

            atom.charge = default_atom.charge
            atom.debye = default_atom.debye

        project.save_object(args.filename + ".def")

        del project

        print "Finished!"
Example #4
0
    def open_project(self, filename):
        # Try to load the project:
        with self.ui_error_handler("An error has occurred.\n Your project was not loaded!"):
            self.model.current_project = Project.load_object(filename, parent=self.model)

        # Set the current filename property and update the title
        self.model.current_filename = filename
        self.update_title()
Example #5
0
    def phaseworker(in_queue, save_queue, stop):
        """
            Parses Phase descriptions into actual objects and passes them
            to the save_queue.
            'stop' should be a threading.Event() that should be toggled
            once all elements have been Queued.
            This way, the worker will only stop once the Queue is really empty,
            and not when it's processing faster than the Queue can be filled. 
        """
        while True:
            try:
                phases_path, phase_descr = in_queue.get(timeout=0.5)
                project = Project()
                phase_lookup = {}
                component_lookup = {}

                for phase_kwargs, code, comp_props in phase_descr:

                    # create phase:
                    G = len(code) / code_length
                    based_on = None
                    if "based_on" in phase_kwargs:
                        based_on = phase_lookup.get(phase_kwargs.pop("based_on"), None)
                    phase = Phase(G=G, parent=project, **phase_kwargs)
                    phase.based_on = based_on
                    phase_lookup[phase.name] = phase

                    # derive upper and lower limits for the codes using code lengths:
                    limits = list(zip(
                        list(range(0, len(code), code_length)),
                        list(range(code_length, len(code) + 1, code_length))
                    ))

                    # create components:
                    phase.components[:] = []
                    for ll, ul in limits:
                        part = code[ll: ul]
                        for component in Component.load_components(aliases[part] % (settings.DATA_REG.get_directory_path("DEFAULT_COMPONENTS") + "/"), parent=phase):
                            component.resolve_json_references()
                            phase.components.append(component)
                            props = comp_props.get(part, {})
                            for prop, value in props.items():
                                if prop == 'linked_with':
                                    value = component_lookup[value]
                                setattr(component, prop, value)

                            component_lookup[part] = component

                # put phases on the save queue:
                phases_path = phases_path % (settings.DATA_REG.get_directory_path("DEFAULT_PHASES") + "/")
                save_queue.put((phases_path, list(phase_lookup.values())))
                # Flag this as finished
                in_queue.task_done()
            except queue.Empty:
                if not stop.is_set():
                    continue
                else:
                    return
Example #6
0
    def new_project(self, *args, **kwargs):
        # Create a new project
        self.model.current_project = Project(parent=self.model)

        # Set the current filename property and update the title
        self.update_title()

        # Show the edit project dialog
        self.view.project.present()
Example #7
0
def _run_gui(project=None):

    # Display a splash screen showing the loading status...
    from pkg_resources import resource_filename # @UnresolvedImport
    from pyxrd.application.splash import SplashScreen
    from pyxrd import __version__

    filename = resource_filename(__name__, "application/icons/pyxrd.png")
    splash = SplashScreen(filename, __version__)

    # Run GUI:
    splash.set_message("Loading GUI ...")

    # Now we can load these:
    from pyxrd.data import settings
    from pyxrd.project.models import Project
    from pyxrd.application.models import AppModel
    from pyxrd.application.views import AppView
    from pyxrd.application.controllers import AppController
    from pyxrd.generic.gtk_tools.gtkexcepthook import plugin_gtk_exception_hook

    filename = settings.ARGS.filename #@UndefinedVariable

    # Check if a filename was passed, if so try to load it
    if filename != "":
        try:
            logging.info("Opening project: %s" % filename)
            project = Project.load_object(filename)
        except IOError:
            logging.info("Could not load project file %s: IOError" % filename)
            # FIXME the user should be informed of this in a dialog...

    # Disable unity overlay scrollbars as they cause bugs with modal windows
    os.environ['LIBOVERLAY_SCROLLBAR'] = '0'
    os.environ['UBUNTU_MENUPROXY'] = ""

    if not settings.DEBUG:
        warnings.filterwarnings(action='ignore', category=Warning)

    # Close splash screen
    if splash: splash.close()

    # Nice GUI error handler:
    gtk_exception_hook = plugin_gtk_exception_hook()

    # setup MVC:
    m = AppModel(project=project)
    v = AppView()
    AppController(m, v, gtk_exception_hook=gtk_exception_hook)

    # Free this before continuing
    del splash

    # lets get this show on the road:
    gtk.main()
def run(args):
    # batch csv file generator for a file containin a number of projects
    if args and args.filename != "":
        project = Project.load_object(args.filename)
        for i, mixture in enumerate(project.mixtures):
            mixture.update_refinement_treestore()
            refs = []
            props = ("d001", "F1", "F2", "F3", "F4", "W1", "P11_or_P22",)
            for node in mixture.refinables.iter_children():
                ref = node.object
                if ref.refinable and ref.prop in props:
                    val = ref.value
                    delta = 0.1
                    if val > 1.0:
                        delta = log(val)
                    ref.value_min = val - delta
                    ref.value_max = val + delta
                    refs.append(ref)

            combs = list(combinations(refs, 2))
            fractions = np.zeros(shape=(len(combs), len(mixture.fractions)))
            rps = np.zeros(shape=(len(combs),))

            print len(combs) * 9, " calculations to be made!"

            for k, (ref1, ref2) in enumerate(combs):
                print "\rProgress: %00.f%%" % float(100.0 * k / len(combs)),
                for i in range(3):
                    for j in range(3):
                        ref1.value = ref1.value_min + float(i) * (ref1.value_max - ref1.value_min) / 2.0
                        ref2.value = ref2.value_min + float(j) * (ref2.value_max - ref2.value_min) / 2.0
                        rp = mixture.optimize()
                        if rp <= 30.0:
                            inv_rp = 30.0 - rp
                            fractions[k, :] = np.array(mixture.fractions, dtype=float)
                            rps[k] = inv_rp
                        # print "\rProgress: %00f%% - iter %d of 25" % (float(100.0 * k / len(combs)), i*5+(j+1))

            print "Phases:"
            print mixture.phases
            print "Unweighted average:"
            print np.average(fractions.transpose(), axis=1, weights=rps)
            print np.std(fractions.transpose() * rps, axis=1) / np.sum(rps)
            print "Weighted average:"
            print np.average(fractions.transpose(), axis=1, weights=rps)
            print np.std(fractions.transpose(), axis=1)
            print "Average Rp value:"
            print np.average(30.0 - rps), "+/-", np.std(30.0 - rps)


        del project

        print "Finished!"
Example #9
0
def setup_project(projectf):
    # Only import these at this point, the cause a considerable increase in
    # memory usage, so if no projectf was passed to improve_solutions, this
    # module does not use more then it needs.
    from pyxrd.data import settings
    settings.apply_runtime_settings()

    from pyxrd.project.models import Project
    type(Project).object_pool.clear()

    f = StringIO()
    f.write(projectf)
    project = Project.load_object(f)
    f.close()

    return project
Example #10
0
def run(args):
    # generates a project file containing the phases as described by the Sybilla XML output:
    if args and args.filename != "":
        # Import:
        project = Project.create_from_sybilla_xml(args.filename)

        # Save this right away:
        project_filename = "%s/%s" % (os.path.dirname(args.filename), os.path.basename(args.filename).replace(".xml", ".pyxrd", 1))
        project.save_object(project_filename)

        # Relaunch processs
        args = [sys.argv[0], project_filename, ]
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]
        os.execv(sys.executable, args)
        sys.exit(0)
Example #11
0
def run(args):
    # batch csv file generator for a file containin a number of projects
    if args and args.filename != "":
        projects = []
        with open(args.filename, 'r') as f:
            for project_file in f:
                project_file = project_file.rstrip()
                print "Parsing: %s" % project_file
                project = Project.load_object(project_file)
                for i, mixture in enumerate(project.mixtures):

                    np.savetxt(
                        "%s/%s" % (os.path.dirname(project_file), os.path.basename(project_file).replace(".pyxrd", "-%d.csv" % i, 1)),
                        get_result_description(mixture), fmt='%s', delimiter=';'
                    )
                del project

        print "Finished!"
Example #12
0
def run(args):
    # generates a project file containing the phases as described by the Sybilla XML output:
    if args and args.filename != "":
        # Import:
        project = Project.create_from_sybilla_xml(args.filename)

        # Save this right away:
        project_filename = "%s/%s" % (os.path.dirname(
            args.filename), os.path.basename(args.filename).replace(
                ".xml", ".pyxrd", 1))

        from pyxrd.file_parsers.json_parser import JSONParser
        JSONParser.write(project, project_filename, zipped=True)

        # Relaunch processs
        args = [
            sys.argv[0],
            project_filename,
        ]
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]
        os.execv(sys.executable, args)
        sys.exit(0)
def run(args):
    """
    This is a simple script that will open a PyXRD project file,
    will run a refinement for a certain mixture, and store the results
    in an overview file and the best solution as a new project file.
    The refinement setup is left unchanged, so be sure you have correctly 
    defined parameter ranges and chosen a good refinement strategy (CMA-ES
    is recommended).
    
    To use this script, launch it using PyXRD's core.py launcher script as:
      python core.py -s pyxrd/scripts/refinement_generator.py "$FILENAME###$I###$J"
    in which: 
        - $FILENAME can be replaced with the absolute path to the actual
          project filename
        - $I is the index of the mixture to refine and
        - $J is the 'trial' number, which is added to the record file and to the
          best solution project file.
        - leave the three # (hashes) where they are, they are used as separators 
        
    You can use this script (e.g. on high-performance computing clusters) to
    run several iterations of the same project. Reasaons why you would want 
    to do this are for benchmarking, checking solution reliability, ... 
    Just change the trial number from e.g. 0 to 49 to have 50 iterations.
    """

    ## TODO:
    ##  - use a uniform distribution of starting solutions:
    ##      xs = np.random.uniform(size=50)
    ##      ys = np.random.uniform(size=50)
    ##      zs = np.random.uniform(size=50)

    ##
    ## When the jobs are submitted, load the project and mixture once,
    ## create the # of staring solutions and store them in a file (using np IO)
    ## Then here we can load them and pick the one we need.
    ##

    if args and args.filename != "":
        logging.info("Proccessing args...")
        project_file, k, mixture_index = tuple(args.filename.split("###", 2))
        base_path = os.path.dirname(args.filename)
        start_solutions_fname = os.path.join(
            base_path,
            "start_solutions %s mixture %s" % (os.path.basename(project_file), mixture_index)
        )
        stop_event = multiprocessing.Event()

        logging.info("Loading project file...")
        project = Project.load_object(project_file)
        logging.info(" ".join(["Running Project", os.path.basename(project_file), "Trial", k]))

        for i, mixture in enumerate(project.mixtures):
            if i == int(mixture_index):

                if B_DO_PROFILING:
                    pr = cProfile.Profile()
                    pr.enable()
                try:
                    with mixture.data_changed.hold():

                        mixture.update_refinement_treestore()
                        mixture.refiner.setup_context(store=True)

                        if int(k) == 0: #First run, create solutions & store for later use:
                            start_solutions = mixture.refiner.context.get_uniform_solutions(50)
                            np.savetxt(start_solutions_fname, start_solutions)
                        else:
                            start_solutions = np.loadtxt(start_solutions_fname)

                        mixture.refiner.context.set_initial_solution(start_solutions[k, ...])
                        mixture.optimizer.optimize()

                        mixture.refiner.refine(stop=stop_event)
                except:
                    raise
                finally:
                    if B_DO_PROFILING:
                        pr.disable()
                        with open("pyxrd_stats", "w+") as f:
                            sortby = 'cumulative'
                            ps = pstats.Stats(pr, stream=f).sort_stats(sortby)
                            ps.print_stats()

                context = mixture.refiner.context

                recordf = os.path.basename(project_file).replace(".pyxrd", "")
                recordf = base_path + "/" + "record#" + str(k) + " " + recordf + " " + mixture.name
                with codecs.open(recordf, 'w', 'utf-8') as f:

                    f.write("################################################################################\n")
                    f.write(recordf + "\n")
                    f.write("Mixture " + str(i) + " and trial " + str(k) + "\n")
                    f.write("Property name, initial, best, min, max" + "\n")
                    for j, ref_prop in enumerate(context.ref_props):
                        line = ", ".join([
                            ref_prop.get_descriptor(),
                            str(context.initial_solution[j]),
                            str(context.best_solution[j]),
                            str(ref_prop.value_min),
                            str(ref_prop.value_max),
                        ])
                        f.write(line + "\n")
                    f.write("################################################################################\n")

                    def write_records(f, record_header, records):
                        f.write(", ".join(record_header) + "\n")
                        for record in records:
                            f.write(", ".join(map(lambda f: "%.7f" % f, record)) + "\n")
                        f.write("################################################################################\n")
                    if context.record_header is not None:
                        write_records(f, context.record_header, context.records)
                    if hasattr(context, "pcma_records"):
                        for record_header, records in context.pcma_records:
                            write_records(f, record_header, records)

                    # Apply found solution and save:
                    context.apply_best_solution()
                    mixture.optimizer.optimize()

                    project_file_output = base_path + "/" + os.path.basename(project_file).replace(".pyxrd", "") + " - mixture %s - trial %s.pyxrd" % (str(i), str(k))
                    project.save_object(file=project_file_output)

        pass # end
Example #14
0
 def setUp(self):
     mock_settings()
     self.project = Project(name="Test Project")
Example #15
0
def run(args):
    project = Project(name="Test")
    project.name = "Test123"
    pass
Example #16
0
 def setUp(self):
     mock_settings()
     self.project = Project(name="TestProject")
     self.mixture = Mixture(name="TestMixture", parent=self.project)
Example #17
0
def create_project_from_sybilla_xml(filename, **kwargs):
    """
        Creates a new project structure from a Sybilla XML file.
        Some information (e.g. the actual XRD pattern) is not present and will
        still need to be imported manually.
    """

    tree = ET.parse(filename)
    root = tree.getroot()
    basename = os.path.basename(filename)

    # Create the project:
    if "name" in kwargs: kwargs.pop("name")
    if "layout_mode" in kwargs: kwargs.pop("layout_mode")
    project = Project(name=basename, layout_mode="FULL", **kwargs)

    # Add a specimen:
    specimen = Specimen(name=basename, parent=project)
    project.specimens.append(specimen)

    # Add a mixture:
    mixture = Mixture(name=basename, auto_run=False, parent=project)
    mixture.add_specimen_slot(specimen, 1.0, 0.0)
    project.mixtures.append(mixture)

    with project.data_changed.ignore():
        with mixture.data_changed.ignore():

            for child in root:
                if child.tag == "basic_params":
                    # Goniometer parameters:
                    step_size = safe_float(child.attrib['step_size'])
                    wavelength = safe_float(child.attrib['lambda']) / 10.0
                    steps = int(1 +
                                (specimen.goniometer.max_2theta -
                                 specimen.goniometer.min_2theta) / step_size)

                    specimen.goniometer.min_2theta = safe_float(
                        child.attrib['min2theta'])
                    specimen.goniometer.max_2theta = safe_float(
                        child.attrib['max2theta'])
                    specimen.goniometer.steps = steps
                    specimen.goniometer.wavelength = wavelength
                elif child.tag == "diffractometer":
                    # Some more goniometer parameters, and specimen parameters:
                    specimen.goniometer.radius = safe_float(
                        child.attrib['gonio_radius'])
                    specimen.goniometer.divergence = safe_float(
                        child.attrib['diverg_slit'])
                    specimen.goniometer.soller1 = safe_float(
                        child.attrib['Soller1'])
                    specimen.goniometer.soller2 = safe_float(
                        child.attrib['Soller2'])
                    specimen.sample_length = safe_float(
                        child.attrib['sample_length'])
                elif child.tag == "content":
                    # Content tag contains 'Mixture' data
                    for xmlPhaseContent in child:
                        name = xmlPhaseContent.attrib['name']
                        fraction = safe_float(
                            xmlPhaseContent.attrib['content']) / 100.
                        mixture.add_phase_slot(name, fraction)
                elif child.tag == "mixture":
                    # Mixture tag corresponds with the phases in the project level,
                    # not an actual Mixture object:
                    for xmlPhase in child:
                        name = xmlPhase.attrib['name']
                        sigma = xmlPhase.attrib['sigma_star']
                        csds = safe_float(
                            xmlPhase.find('distribution').attrib['Tmean'])
                        G = 1
                        R = 0
                        W = [
                            1.0,
                        ]
                        if xmlPhase.attrib['type'] != 'mono':
                            prob = xmlPhase.find('probability')
                            G = int(prob.attrib['no_of_comp'])
                            R = int(prob.attrib['R'])

                        # create phase and add to project:
                        phase = Phase(name=name,
                                      sigma_star=sigma,
                                      G=G,
                                      R=R,
                                      parent=project)
                        phase.CSDS_distribution.average = csds
                        project.phases.append(phase)

                        # set probability:
                        if R == 0 and G != 1:
                            xmlW = prob.find('W')
                            W = np.array([
                                float(
                                    int(
                                        safe_float(xmlW.attrib[
                                            string.ascii_lowercase[i]]) *
                                        1000.)) / 1000. for i in range(G)
                            ])
                            for i in range(G - 1):
                                setattr(phase.probabilities, "F%d" % (i + 1),
                                        W[i] / np.sum(W[i:]))
                        if R == 1 and G == 2:
                            pass  # TODO
                        # ... TODO other probs

                        # parse components:
                        for i, layer in enumerate(
                                xmlPhase.findall("./layer_and_edge/layer")):

                            component = phase.components[i]
                            component.name = layer.attrib['name']

                            component.d001 = safe_float(
                                layer.attrib['d_spacing']) / 10.0
                            component.default_c = safe_float(
                                layer.attrib['d_spacing']) / 10.0
                            component.delta_c = safe_float(
                                layer.attrib['d_spacing_delta']) / 10.0

                            component.ucp_b.value = 0.9

                            component.ucp_a.factor = 0.57735
                            component.ucp_a.prop = (component, 'cell_b')
                            component.ucp_a.enabled = True

                            atom_type_map = {
                                # "NH4": "FIXME"
                                "K": "K1+",
                                "O": "O1-",
                                "Si": "Si2+",
                                "OH": "OH1-",
                                "Fe": "Fe1.5+",
                                "Al": "Al1.5+",
                                "Mg": "Mg1+",
                                "H2O": "H2O",
                                "Gly": "Glycol",
                                "Ca": "Ca2+",
                                "Na": "Na1+",
                            }

                            # add atoms:
                            fe_atom = None
                            encountered_oxygen = False
                            for atom in layer.findall("atom"):
                                atom_type_name = atom_type_map.get(
                                    atom.attrib['type'], None)
                                if atom_type_name:
                                    if atom_type_name == "O1-":
                                        # From this point we're dealing with layer atoms
                                        encountered_oxygen = True
                                    atom = Atom(
                                        name=atom.attrib['type'],
                                        default_z=safe_float(
                                            atom.attrib['position']) / 10.0,
                                        pn=safe_float(atom.attrib['content']),
                                        atom_type_name=atom_type_name,
                                        parent=component)
                                    if encountered_oxygen:
                                        component.layer_atoms.append(atom)
                                    else:
                                        component.interlayer_atoms.append(atom)
                                    atom.resolve_json_references()
                                    # Assume this is the octahedral iron...
                                    if encountered_oxygen and atom_type_name == "Fe1.5+":
                                        fe_atom = atom

                            # Set the atom relation
                            if fe_atom is not None:
                                component.ucp_b.constant = 0.9
                                component.ucp_b.factor = 0.0043
                                component.ucp_b.prop = (fe_atom, 'pn')
                                component.ucp_b.enabled = True

                pass  # end of if
            pass  # end of for

            # Map phases onto mixture names:
            for phase in project.phases:
                for slot, phase_name in enumerate(mixture.phases):
                    if phase.name == phase_name:
                        mixture.set_phase(0, slot, phase)

    return project
Example #18
0
 def test_parent(self):
     parent_project = Project(name="Parent2")
     self.mixture.parent = parent_project
     self.assertEqual(self.mixture.parent, parent_project)
Example #19
0
def run(args):
    project = Project(name="Test")
    project.name = "Test123"
    pass
Example #20
0
 def setUp(self):
     self.project = Project.load_object(resource_filename("test.test_mixture", "test refinement.pyxrd"))
     self.mixture = self.project.mixtures[0]