コード例 #1
0
def add_snl(mat, new_style_mat):
    snl = new_style_mat.get("snl", None)
    mat["snl"] = copy.deepcopy(mat["structure"])
    if snl:
        mat["snl"].update(snl)
    else:
        mat["snl"] = StructureNL(Structure.from_dict(mat["structure"]),
                                 []).as_dict()
        mat["snl"]["about"].update(mp_default_snl_fields)

    mat["snl_final"] = mat["snl"]
    mat["icsd_ids"] = [
        int(i) for i in get(mat["snl"], "about._db_ids.icsd_ids", [])
    ]
    mat["pf_ids"] = get(mat["snl"], "about._db_ids.pf_ids", [])

    # Extract tags from remarks by looking for just nounds and adjectives
    mat["exp"] = {"tags": []}
    for remark in mat["snl"]["about"].get("_tags", []):
        tokens = set(tok[1] for tok in nltk.pos_tag(nltk.word_tokenize(remark),
                                                    tagset="universal"))
        if len(tokens.intersection({"ADV", "ADP", "VERB"})) == 0:
            mat["exp"]["tags"].append(remark)
コード例 #2
0
def update_current_schedule():
    schedule_data = fetch_current_schedule()
    if not schedule_data:
        return

    game_dates = get(schedule_data, "leagueSchedule.gameDates")
    games = flat_map(game_dates, lambda game_date: get(game_date, "games"))
    reg_season_games = filter(lambda game: get(game, "weekNumber") > 0, games)
    game_data = list(
        map(
            lambda game: {
                **omit(game, [
                    "broadcasters", "homeTeam", "awayTeam", "pointsLeaders"
                ]),
                "homeTeamId":
                get(game, "homeTeam.teamId"),
                "awayTeamId":
                get(game, "awayTeam.teamId"),
            },
            reg_season_games,
        ))
    primary_keys = ["gameId"]
    update_data_table_from_dicts("game_schedule", game_data, primary_keys)
コード例 #3
0
def getValidMethod(obj):
    '''
    获取请求方法的有效方法类型
    :param obj: 请求对象
    :return: 返回有效的请求方法类型名称 或者 None
    '''
    methods = ['get', 'post', 'put', 'delete', 'patch']

    valids = []

    for method in methods:
        if objects.get(obj, method) != None:
            valids.append(method)
    return valids
コード例 #4
0
def create_confirmation_email_dict(response, confirmationEmailInfo):
    # Creates dict with confirmation email info.
    if not confirmationEmailInfo:
        return
    templateText = get(
        confirmationEmailInfo, "template.html", "Confirmation Email Sample Text"
    )
    msgBody = fill_string_from_template(response, templateText)
    addCSS = False

    toField = confirmationEmailInfo["toField"]
    if type(toField) is not list:
        toField = [toField]

    # pre-process attachment templates
    attachments = [
        {
            "fileName": item.get("fileName", "attachment.pdf"),
            "contents": convert_html_to_pdf(
                fill_string_from_template(response, item["template"]["html"])
            ),
        }
        for item in confirmationEmailInfo.get("attachments", [])
    ]

    emailOptions = dict(
        toEmail=[get(response.value, i) for i in toField],
        fromEmail=confirmationEmailInfo.get("from", "*****@*****.**"),
        fromName=confirmationEmailInfo.get("fromName", "Webmaster"),
        subject=confirmationEmailInfo.get("subject", "Confirmation Email"),
        bccEmail=confirmationEmailInfo.get("bcc", ""),
        ccEmail=confirmationEmailInfo.get("cc", ""),
        replyToEmail=confirmationEmailInfo.get("replyTo", ""),
        msgBody=msgBody,
        attachments=attachments,
    )
    return emailOptions
コード例 #5
0
ファイル: materials.py プロジェクト: mattmcdermott/emmet
    def filter_and_group_tasks(self, tasks):
        """
        Groups tasks by structure matching
        """

        filtered_tasks = [
            t for t in tasks if task_type(t["orig_inputs"]) in self.allowed_tasks
        ]

        structures = []

        for idx, t in enumerate(filtered_tasks):
            s = Structure.from_dict(t["output"]["structure"])
            s.index = idx
            total_mag = get(t, "calcs_reversed.0.output.outcar.total_magnetization", 0)
            s.total_magnetization = total_mag if total_mag else 0
            # a fix for very old tasks that did not report site-projected magnetic moments
            # so that we can group them appropriately
            if (
                ("magmom" not in s.site_properties)
                and (get(t, "input.parameters.ISPIN", 1) == 2)
                and has(t, "input.parameters.MAGMOM")
            ):
                # TODO: map input structure sites to output structure sites
                s.add_site_property("magmom", t["input"]["parameters"]["MAGMOM"])
            structures.append(s)

        grouped_structures = group_structures(
            structures,
            ltol=self.ltol,
            stol=self.stol,
            angle_tol=self.angle_tol,
            separate_mag_orderings=self.separate_mag_orderings,
        )

        for group in grouped_structures:
            yield [filtered_tasks[struc.index] for struc in group]
コード例 #6
0
def fill_paymentMethods_with_data(paymentMethods, form_data):
    new_paymentMethods = {}
    for paymentMethodName, paymentMethod in paymentMethods.items():
        new_paymentMethods[paymentMethodName] = {}
        for key, value in paymentMethod.items():
            if type(value) is str:
                value = [value]
            if type(value) is list:
                value = [
                    (get(form_data, x[1:], ".") if x.startswith("$") else x)
                    for x in value
                ]
                value = "".join(value)
            new_paymentMethods[paymentMethodName][key] = value
    return new_paymentMethods
コード例 #7
0
ファイル: mp_website.py プロジェクト: schubertYann/emmet
def old_style_mat(new_mat):
    """
    Creates the base document for the old MP mapidoc style from the new document structure
    """

    mat = {}
    for mp, new_key in mp_conversion_dict.items():
        if has(new_mat, new_key):
            set_(mat, mp, get(new_mat, new_key))

    mat["is_orderd"] = True
    mat["is_compatible"] = True

    struc = Structure.from_dict(mat["structure"])
    mat["oxide_type"] = oxide_type(struc)
    mat["reduced_cell_formula"] = struc.composition.as_dict()
    mat["full_formula"] = "".join(struc.formula.split())
    vals = sorted(mat["reduced_cell_formula"].values())
    mat["anonymous_formula"] = {
        string.ascii_uppercase[i]: float(vals[i])
        for i in range(len(vals))
    }

    set_(mat, "pseudo_potential.functional", "PBE")

    set_(mat, "pseudo_potential.labels", [
        p["titel"].split()[1]
        for p in get(new_mat, "calc_settings.potcar_spec")
    ])
    mat["ntask_ids"] = len(get(new_mat, "task_ids"))
    set_(mat, "pseudo_potential.pot_type", "paw")
    add_bv_structure(mat)
    add_blessed_tasks(mat, new_mat)
    add_cifs(mat)

    return mat
コード例 #8
0
ファイル: renderers_v2.py プロジェクト: rm-rfadmin/lightning
    def inner_hand(data, index):
        key = key_split[index]
        value = objects.get(data, key)

        if index == (len(key_split) - 1):
            if value is not None:
                result.append(str(value))
            return

        if isinstance(value, list):
            k_index = index + 1
            for item in value:
                inner_hand(item, k_index)
        else:
            inner_hand(value, index + 1)
コード例 #9
0
ファイル: mapper.py プロジェクト: echo-liang/RSBC-DataHub-API
 def _create_one_to_one_record(table_name, message) -> dict:
     """
     Build a dictionary with a list for the field names
      (columns) and the associated values
     :param table_name:
     :param message:
     :return:
     """
     table = {'columns': [], 'values': [], 'table': table_name['name']}
     for field in table_name['fields']:
         value = objects.get(message, field['json_name'])
         if value:
             table['columns'].append(field['destination_name'])
             table['values'].append(value)
     return table
コード例 #10
0
ファイル: filter.py プロジェクト: syllogy/cfripper
def get_implemented_filter_function(function_name: str,
                                    debug: bool) -> Callable:
    def param_resolver(f):
        def wrap(*args, **kwargs):
            calculated_parameters = [arg(kwargs) for arg in args]
            result = f(*calculated_parameters, **kwargs)
            if debug:
                logger.debug(
                    f"{function_name}({', '.join(str(x) for x in calculated_parameters)}) -> {result}"
                )
            return result

        return wrap

    implemented_filter_functions = {
        "eq":
        param_resolver(lambda a, b, **kwargs: a == b),
        "ne":
        param_resolver(lambda a, b, **kwargs: a != b),
        "lt":
        param_resolver(lambda a, b, **kwargs: a < b),
        "gt":
        param_resolver(lambda a, b, **kwargs: a > b),
        "le":
        param_resolver(lambda a, b, **kwargs: a <= b),
        "ge":
        param_resolver(lambda a, b, **kwargs: a >= b),
        "not":
        param_resolver(lambda a, **kwargs: not a),
        "or":
        lambda *args, **kwargs: any(arg(kwargs) for arg in args),
        "and":
        lambda *args, **kwargs: all(arg(kwargs) for arg in args),
        "in":
        param_resolver(lambda a, b, **kwargs: a in b),
        "regex":
        param_resolver(lambda *args, **kwargs: bool(re.match(*args))),
        "regex:ignorecase":
        param_resolver(
            lambda *args, **kwargs: bool(re.match(*args, re.IGNORECASE))),
        "exists":
        param_resolver(lambda a, **kwargs: a is not None),
        "empty":
        param_resolver(lambda *args, **kwargs: len(args) == 0),
        "ref":
        param_resolver(lambda param_name, **kwargs: get(kwargs, param_name)),
    }
    return implemented_filter_functions[function_name]
コード例 #11
0
    def search_bulk(self, keywords=None, author=None):
        """Build a list of all books for given author/keyword combination"""
        rv = list()

        if not keywords and not author:
            raise KeyError("No value for search!")

        while self.current_result <= self.max_results:
            r = self.search_books(keywords=keywords,
                                  author=author,
                                  index=self.current_result)
            if get(r, 'items'):
                for book in r['items']:
                    rv.append(book)
                    self.current_result += 1
            else:
                return rv  # as there are no more results, we exit the while loop
        return rv
コード例 #12
0
def update_response_path(response, path, value, response_base_path):
    from ..main import app
    existing_value = get(getattr(response, response_base_path), path, value)
    if type(value) is str:
        if type(existing_value) is bool and value.lower() in ("true", "false"):
            value = value.lower() == "true"
        elif type(existing_value) is float and value.isdigit():
            value = float(value)
        elif type(existing_value) is int and value.isdigit():
            value = int(value)
    set_(getattr(response, response_base_path), path, value)
    response.update_trail.append(
        UpdateTrailItem(path=path,
                        old_value=existing_value,
                        new_value=value,
                        date=datetime.datetime.now(),
                        user=app.get_current_user_id(),
                        response_base_path=response_base_path))
コード例 #13
0
def buildTop(data):
    '''
    构建swagger顶层目录结构
    :param data: swagger原始返回的信息
    :return:返回顶层元组信息
    '''
    try:
        dataObj = json.loads(data)
        tags = objects.get(dataObj, 'tags', [])
        tops = arrays.compact(tags)
        tops = arrays.mapcat(
            tops,
            lambda item: objects.assign(objects.clone(item), {'child': []}))

        return tops
    except:
        sysLog.warn('JSON处理失败,请确认地址是否正确')
        return []
コード例 #14
0
def convert_mpworks_to_atomate(mpworks_doc, update_mpworks=True):
    """
    Function to convert an mpworks document into an atomate
    document, uses schema above and a few custom cases

    Args:
        mpworks_doc (dict): mpworks task document
        update_mpworks (bool): flag to indicate that mpworks schema
            should be updated to final MPWorks version
    """
    if update_mpworks:
        update_mpworks_schema(mpworks_doc)

    atomate_doc = {}
    for key_mpworks, key_atomate in settings['task_conversion_keys'].items():
        val = get(mpworks_doc, key_mpworks)
        set_(atomate_doc, key_atomate, val)

    # Task type
    atomate_doc["task_label"] = settings['task_label_conversions'].get(
        mpworks_doc["task_type"])

    # calculations
    atomate_doc["calcs_reversed"] = mpworks_doc["calculations"][::-1]

    # anonymous formula
    comp = Composition(atomate_doc['composition_reduced'])
    atomate_doc["formula_anonymous"] = comp.anonymized_formula

    # deformation matrix and original_task_id
    if "deformation_matrix" in mpworks_doc:
        # Transpose this b/c of old bug, should verify in doc processing
        defo = mpworks_doc["deformation_matrix"]
        if isinstance(defo, str):
            defo = convert_string_deformation_to_list(defo)
        defo = np.transpose(defo).tolist()
        set_(atomate_doc, "transmuter.transformations",
             ["DeformStructureTransformation"])
        set_(atomate_doc, "transmuter.transformation_params",
             [{
                 "deformation": defo
             }])

    return atomate_doc
コード例 #15
0
ファイル: __init__.py プロジェクト: gabrielsm/domain_reader
    def _get_insert_sql(self, table, entity, fields, from_id):
        if from_id:
            entity['from_id'] = from_id

        entity['branch'] = objects.get(entity, '_metadata.branch')
        columns = [
            field['column'] for field in fields if field['name'] in entity
        ]
        values = ['%s' for field in fields if field['name'] in entity]
        params = tuple(entity[field['name']] for field in fields
                       if field['name'] in entity)
        return {
            'query':
            self.QUERIES['insert'].format(table=table,
                                          values=str.join(',', values),
                                          columns=str.join(',', columns)),
            'params':
            params
        }
コード例 #16
0
ファイル: mp_website.py プロジェクト: richardtran415/emmet
def old_style_mat(new_mat):

    mat = {}
    for mp, new_key in mp_conversion_dict.items():
        if has(new_mat, new_key):
            set_(mat, mp, get(new_mat, new_key))

    mat["is_orderd"] = True
    mat["is_compatible"] = True

    struc = Structure.from_dict(mat["structure"])
    mat["oxide_type"] = oxide_type(struc)
    mat["reduced_cell_formula"] = struc.composition.as_dict()
    mat["full_formula"] = "".join(struc.formula.split())
    vals = sorted(mat["reduced_cell_formula"].values())
    mat["anonymous_formula"] = {
        string.ascii_uppercase[i]: float(vals[i])
        for i in range(len(vals))
    }

    set_(mat, "original_task_id", get(new_mat, "material_id"))
    set_(mat, "ntask_ids", len(get(new_mat, "task_ids")))

    set_(mat, "input.incar", get(new_mat,
                                 "inputs.structure_optimization.incar"))
    set_(mat, "input.kpoints",
         get(new_mat, "inputs.structure_optimization.kpoints"))
    set_(mat, "encut", get(new_mat,
                           "inputs.structure_optimization.incar.ENCUT"))
    mat["pseudo_potential"] = {
        "pot_type":
        "paw",
        "labels":
        get(new_mat, "input.structure_optimization.potcar.symbols"),
        "functional":
        get(new_mat, "input.structure_optimization.potcar.functional")
    }

    return mat
コード例 #17
0
ファイル: snls.py プロジェクト: richardtran415/emmet
    def update_targets(self, items):
        """
        Inserts the new task_types into the task_types collection

        """
        snls = []

        for snl_dict in filter(None, items):
            for mat_id, snl_list in snl_dict.items():
                snl = sorted(
                    snl_list, key=lambda x: StructureNL.from_dict(x).created_at)[0]
                icsd_ids = [get(snl, "about._icsd.icsd_id")
                            for snl in snl_list if has(snl, "about._icsd")]
                snls.append(
                    {self.snls.key: mat_id, "snl": snl, "icsd_ids": icsd_ids})

        if len(snls) > 0:
            self.snls.update(snls)
        else:
            self.logger.info("No items to update")
コード例 #18
0
ファイル: mp_website.py プロジェクト: jerrymlin/emmet
def add_thermo(mat, new_style_mat):
    """
    Add's the thermo values in with sandboxing
    """
    if "thermo_docs" not in new_style_mat:
        mat["deprecated"] = True
        
    if not mat["deprecated"]:
        thermo = new_style_mat["thermo_docs"]

        if "core" in mat["sbxn"]:
            main_sbx = "core"
        else:
            main_sbx = mat["sbxn"][0]

        # Get the primary document and set in mat document
        core_thermo = next(d for d in thermo if main_sbx in d["_sbxn"])

        mat["e_above_hull"] = core_thermo["thermo"]["e_above_hull"]
        mat["formation_energy_per_atom"] = core_thermo["thermo"][
            "formation_energy_per_atom"
        ]
        if "decomposes_to" in core_thermo["thermo"]:
            mat["decomposes_to"] = core_thermo["thermo"]["decomposes_to"]

        sbxd = {}
        sandbox_props = {
            "e_above_hull": "thermo.e_above_hull",
            "decomposes_to": "thermo.decomposes_to",
        }
        for doc in thermo:
            for sbx in doc["_sbxn"]:
                sbx_d = {
                    k: get(doc, v) for k, v in sandbox_props.items() if has(doc, v)
                }
                sbx_d["id"] = sbx
                sbxd[sbx] = sbx_d

        mat["sbxd"] = list(sbxd.values())
コード例 #19
0
ファイル: __init__.py プロジェクト: gabrielsm/domain_reader
    def _get_update_sql(self,
                        entity_id,
                        table,
                        entity,
                        fields,
                        change_track,
                        from_id=None):
        if from_id:
            entity['from_id'] = from_id

        entity['deleted'] = change_track == 'destroy'
        entity['branch'] = objects.get(entity, '_metadata.branch')
        values = [f"{field['column']}=%s" for field in fields]
        params = tuple(
            entity[field['name']] if field['name'] in entity else None
            for field in fields)
        params += (entity_id, )  # entity id parameter
        return {
            'query':
            self.QUERIES['update'].format(table=table,
                                          values=str.join(',', values)),
            'params':
            params
        }
コード例 #20
0
def get_elastic_analysis(opt_task, defo_tasks):
    """
    Performs the analysis of opt_tasks and defo_tasks necessary for
    an elastic analysis

    Args:
        opt_task: task doc corresponding to optimization
        defo_tasks: task_doc corresponding to deformations

    Returns:
        elastic document with fitted elastic tensor and analysis

    """
    elastic_doc = {"warnings": []}
    opt_struct = Structure.from_dict(opt_task['output']['structure'])
    input_struct = Structure.from_dict(opt_task['input']['structure'])
    # For now, discern order (i.e. TOEC) using parameters from optimization
    # TODO: figure this out more intelligently
    diff = get(opt_task, "input.incar.EDIFFG", 0)
    order = 3 if np.isclose(diff, -0.001) else 2
    explicit, derived = process_elastic_calcs(opt_task, defo_tasks)
    all_calcs = explicit + derived
    stresses = [c.get("cauchy_stress") for c in all_calcs]
    pk_stresses = [c.get("pk_stress") for c in all_calcs]
    strains = [c.get("strain") for c in all_calcs]
    elastic_doc['calculations'] = all_calcs
    vstrains = [s.zeroed(0.002).voigt for s in strains]
    if np.linalg.matrix_rank(vstrains) == 6:
        if order == 2:
            et_fit = legacy_fit(strains, stresses)
        elif order == 3:
            # Test for TOEC
            if len(strains) < 70:
                logger.info("insufficient valid strains for {} TOEC".format(
                    opt_task['formula_pretty']))
                return None
            eq_stress = -0.1 * Stress(opt_task['output']['stress'])
            # strains = [s.zeroed(0.0001) for s in strains]
            # et_expansion = pdb_function(ElasticTensorExpansion.from_diff_fit,
            #     strains, pk_stresses, eq_stress=eq_stress, tol=1e-5)
            et_exp_raw = ElasticTensorExpansion.from_diff_fit(
                strains, pk_stresses, eq_stress=eq_stress, tol=1e-6)
            et_exp = et_exp_raw.voigt_symmetrized.convert_to_ieee(opt_struct)
            et_exp = et_exp.round(1)
            et_fit = ElasticTensor(et_exp[0])
            # Update elastic doc with TOEC stuff
            tec = et_exp.thermal_expansion_coeff(opt_struct, 300)
            elastic_doc.update({
                "elastic_tensor_expansion":
                elastic_sanitize(et_exp),
                "elastic_tensor_expansion_original":
                elastic_sanitize(et_exp_raw),
                "thermal_expansion_tensor":
                tec,
                "average_linear_thermal_expansion":
                np.trace(tec) / 3
            })
        et = et_fit.voigt_symmetrized.convert_to_ieee(opt_struct)
        vasp_input = opt_task['input']
        if 'structure' in vasp_input:
            vasp_input.pop('structure')
        completed_at = max([d['completed_at'] for d in defo_tasks])
        elastic_doc.update({
            "optimization_task_id":
            opt_task['task_id'],
            "optimization_dir_name":
            opt_task['dir_name'],
            "cauchy_stresses":
            stresses,
            "strains":
            strains,
            "elastic_tensor":
            elastic_sanitize(et.zeroed(0.01).round(0)),
            # Convert compliance to 10^-12 Pa
            "compliance_tensor":
            elastic_sanitize(et.compliance_tensor * 1000),
            "elastic_tensor_original":
            elastic_sanitize(et_fit),
            "optimized_structure":
            opt_struct,
            "spacegroup":
            input_struct.get_space_group_info()[0],
            "input_structure":
            input_struct,
            "completed_at":
            completed_at,
            "optimization_input":
            vasp_input,
            "order":
            order,
            "pretty_formula":
            opt_struct.composition.reduced_formula
        })
        # Add magnetic type
        mag = CollinearMagneticStructureAnalyzer(opt_struct).ordering.value
        elastic_doc['magnetic_type'] = mag_types[mag]
        try:
            prop_dict = et.get_structure_property_dict(opt_struct)
            prop_dict.pop('structure')
        except ValueError:
            logger.debug("Negative K or G found, structure property "
                         "dict not computed")
            prop_dict = et.property_dict
        for k, v in prop_dict.items():
            if k in ['homogeneous_poisson', 'universal_anisotropy']:
                prop_dict[k] = np.round(v, 2)
            else:
                prop_dict[k] = np.round(v, 0)
        elastic_doc.update(prop_dict)
        # Update with state and warnings
        state, warnings = get_state_and_warnings(elastic_doc)
        elastic_doc.update({"state": state, "warnings": warnings})
        # TODO: add kpoints params?
        return elastic_doc
    else:
        logger.info("insufficient valid strains for {}".format(
            opt_task['formula_pretty']))
        return None
コード例 #21
0
ファイル: utils.py プロジェクト: masnec/gulis-processor
def querySelector(tree, css):
    doms = querySelectorAll(tree, css)
    return get(doms, 0)
コード例 #22
0
ファイル: mp_website.py プロジェクト: jerrymlin/emmet
def add_bonds(mat, new_style_mat):
    if get("bonds.successful", new_style_mat, False):
        mat["bonds"] = get("bonds.summary", new_style_mat)
コード例 #23
0
os.environ["TABLE_PREFIX"] = "cff_prod"
from chalicelib.main import TABLES
from chalicelib.util import get_all_responses
from boto3.dynamodb.conditions import Key
import boto3
import json
from pynliner import Pynliner
from pydash.objects import get
import datetime

formId = "31571110-483c-4b72-b4b8-5b1ce0b9348b"

client = boto3.client("ses")
responses = boto3.resource('dynamodb').Table("cff_prod.responses")

print("Querying all responses...")
responses = get_all_responses(KeyConditionExpression=Key('formId').eq(formId),
                              FilterExpression=Key('PAID').eq(True))
print("Got responses.")

used_bibs = []
duplicate_bibs = []
for rNum, response in enumerate(responses):
    print(f"=== Starting response number {rNum} ===")
    for i, participant in enumerate(get(response, "value.participants", [])):
        oldbib = str(get(participant, "bib_number"))
        if oldbib in used_bibs and oldbib != 'None':
            duplicate_bibs.append(oldbib)
        used_bibs.append(oldbib)
print("Done! Duplicates:")
print(duplicate_bibs)
コード例 #24
0
ファイル: parse_outputs.py プロジェクト: shyamd/MatMethods
    def run_task(self, fw_spec):
        # get the directory that contains the VASP dir to parse
        calc_dir = os.getcwd()
        if "calc_dir" in self:
            calc_dir = self["calc_dir"]
        elif self.get("calc_loc"):
            calc_dir = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"])["path"]

        # parse the VASP directory
        logger.info("PARSING DIRECTORY: {}".format(calc_dir))

        drone = VaspDrone(additional_fields=self.get("additional_fields"),
                          parse_dos=self.get("parse_dos", False),
                          bandstructure_mode=self.get("bandstructure_mode", False),
                          parse_chgcar=self.get("parse_chgcar", False),
                          parse_aeccar=self.get("parse_aeccar", False))

        # assimilate (i.e., parse)
        task_doc = drone.assimilate(calc_dir)

        # Check for additional keys to set based on the fw_spec
        if self.get("fw_spec_field"):
            task_doc.update(fw_spec[self.get("fw_spec_field")])

        # get the database connection
        db_file = env_chk(self.get('db_file'), fw_spec)

        # db insertion or taskdoc dump
        if not db_file:
            with open("task.json", "w") as f:
                f.write(json.dumps(task_doc, default=DATETIME_HANDLER))
        else:
            mmdb = VaspCalcDb.from_db_file(db_file, admin=True)
            t_id = mmdb.insert_task(
                task_doc, use_gridfs=self.get("parse_dos", False)
                or bool(self.get("bandstructure_mode", False))
                or self.get("parse_chgcar", False)
                or self.get("parse_aeccar", False))
            logger.info("Finished parsing with task_id: {}".format(t_id))

        defuse_children = False
        if task_doc["state"] != "successful":
            defuse_unsuccessful = self.get("defuse_unsuccessful",
                                           DEFUSE_UNSUCCESSFUL)
            if defuse_unsuccessful is True:
                defuse_children = True
            elif defuse_unsuccessful is False:
                pass
            elif defuse_unsuccessful == "fizzle":
                raise RuntimeError(
                    "VaspToDb indicates that job is not successful "
                    "(perhaps your job did not converge within the "
                    "limit of electronic/ionic iterations)!")
            else:
                raise RuntimeError("Unknown option for defuse_unsuccessful: "
                                   "{}".format(defuse_unsuccessful))

        task_fields_to_push = self.get("task_fields_to_push", None)
        update_spec = {}
        if task_fields_to_push:
            if isinstance(task_fields_to_push, dict):
                for key, path_in_task_doc in task_fields_to_push.items():
                    if has(task_doc, path_in_task_doc):
                        update_spec[key] = get(task_doc, path_in_task_doc)
                    else:
                        logger.warn("Could not find {} in task document. Unable to push to next firetask/firework".format(path_in_task_doc))
            else:
                raise RuntimeError("Inappropriate type {} for task_fields_to_push. It must be a "
                                   "dictionary of format: {key: path} where key refers to a field "
                                   "in the spec and path is a full mongo-style path to a "
                                   "field in the task document".format(type(task_fields_to_push)))

        return FWAction(stored_data={"task_id": task_doc.get("task_id", None)},
                        defuse_children=defuse_children, update_spec=update_spec)
コード例 #25
0
def update_ccavenue_hash(formId, ccavenuePaymentMethodInfo, response):
    from ...main import app

    def fill_ccavenue_paymentinfo(key):
        value = ccavenuePaymentMethodInfo.get(key)
        return fill_string_from_template(response, value) if value else ""

    merchant_id = ccavenuePaymentMethodInfo["merchant_id"]

    config = CCAvenueConfig.objects.get({"merchant_id": merchant_id})
    if not config:
        raise Exception(
            f"CCAvenue config not found for merchant id: {merchant_id}.")
    """en - English
  hi - Hindi
  gu - Gujarati
  mr - Marathi
  bn - Bengali"""
    responseId = str(response.id)
    orderId = str(ObjectId())
    data = {
        "merchant_id":
        merchant_id,
        "order_id":
        orderId,
        "currency":
        response.paymentInfo["currency"],
        "amount":
        str(
            Decimal(response.paymentInfo["total"]) -
            Decimal(response.amount_paid)),
        "redirect_url":
        app.get_url(f"responses/{responseId}/ccavenueResponseHandler"),
        "cancel_url":
        "http://www.chinmayamission.com",  # todo: fix this.
        "language":
        "en",
        "billing_name":
        fill_ccavenue_paymentinfo("billing_name"),
        "billing_address":
        fill_ccavenue_paymentinfo("billing_address"),
        "billing_city":
        fill_ccavenue_paymentinfo("billing_city"),
        "billing_state":
        fill_ccavenue_paymentinfo("billing_state"),
        "billing_zip":
        fill_ccavenue_paymentinfo("billing_zip"),
        "billing_country":
        fill_ccavenue_paymentinfo("billing_country"),
        "billing_tel":
        fill_ccavenue_paymentinfo("billing_tel"),
        "billing_email":
        fill_ccavenue_paymentinfo("billing_email"),
        "merchant_param1":
        formId,
        "merchant_param2":
        responseId,
        "merchant_param3":
        get(
            ccavenuePaymentMethodInfo,
            "redirectUrl",
            ccavenuePaymentMethodInfo.get("redirectUrl",
                                          "http://www.chinmayamission.com"),
        ),
    }
    sub_account_id = ccavenuePaymentMethodInfo.get("sub_account_id")
    if sub_account_id is not None:
        data["sub_account_id"] = sub_account_id
    # "delivery_name": "test",
    # "delivery_address": "test",
    # "delivery_city": "test",
    # "delivery_state": "test",
    # "delivery_zip": "test",
    # "delivery_country": "test",
    # "delivery_tel": "test",
    # "merchant_param1": "test",
    # "merchant_param2": "test",
    # "merchant_param3": "test",
    # "merchant_param4": "test",
    # "merchant_param5": "test",
    # "integration_type": "test",
    # "promo_code": "test",
    # "customer_identifier": "test"
    ccavenuePaymentMethodInfo["encRequest"] = encrypt(
        data, config.SECRET_working_key)
    ccavenuePaymentMethodInfo["access_code"] = config.access_code
    ccavenuePaymentMethodInfo["merchant_id"] = merchant_id
    return ccavenuePaymentMethodInfo
コード例 #26
0
def group_by_material_id(materials_dict,
                         docs,
                         structure_key='structure',
                         tol=1e-6,
                         loosen=True,
                         structure_matcher=None):
    """
    Groups a collection of documents by material id
    as found in a materials collection

    Args:
        materials_dict (dict): dictionary of structures keyed by task_id
        docs ([dict]): list of documents
        tol (float): tolerance for lattice grouping
        loosen (bool): whether or not to loosen criteria if no matches are
            found
        structure_key (string): mongo-style key of documents where structures
            are contained (e. g. input.structure or output.structure)
        structure_matcher (StructureMatcher): structure
            matcher for finding equivalent structures

    Returns:
        documents grouped by task_id from the materials
        collection
    """
    # Structify all input structures
    materials_dict = {
        mp_id: Structure.from_dict(struct)
        for mp_id, struct in materials_dict.items()
    }
    # Get magnetic phases
    mags = {}
    # TODO: refactor this with data from materials collection?
    for mp_id, structure in materials_dict.items():
        mag = CollinearMagneticStructureAnalyzer(structure).ordering.value
        mags[mp_id] = mag_types[mag]
    docs_by_mp_id = {}
    for doc in docs:
        sm = structure_matcher or StructureMatcher(
            comparator=ElementComparator())
        structure = Structure.from_dict(get(doc, structure_key))
        input_sg_symbol = SpacegroupAnalyzer(structure,
                                             0.1).get_space_group_symbol()
        # Iterate over all candidates until match is found
        matches = {
            c_id: candidate
            for c_id, candidate in materials_dict.items()
            if sm.fit(candidate, structure)
        }
        niter = 0
        if not matches:
            # First try with conventional structure then loosen match criteria
            convs = {
                c_id:
                SpacegroupAnalyzer(candidate,
                                   0.1).get_conventional_standard_structure()
                for c_id, candidate in materials_dict.items()
            }
            matches = {
                c_id: candidate
                for c_id, candidate in materials_dict.items()
                if sm.fit(convs[c_id], structure)
            }
            while len(matches) < 1 and niter < 4 and loosen:
                logger.debug("Loosening sm criteria")
                sm = StructureMatcher(sm.ltol * 2,
                                      sm.stol * 2,
                                      sm.angle_tol * 2,
                                      primitive_cell=False)
                matches = {
                    c_id: candidate
                    for c_id, candidate in materials_dict.items()
                    if sm.fit(convs[c_id], structure)
                }
                niter += 1
        if matches:
            # Get best match by spacegroup, then mag phase, then closest density
            mag = doc['magnetic_type']

            def sort_criteria(m_id):
                dens_diff = abs(matches[m_id].density - structure.density)
                sg = matches[m_id].get_space_group_info(0.1)[0]
                mag_id = mags[m_id]
                # prefer explicit matches, allow non-mag materials match with FM tensors
                if mag_id == mag:
                    mag_match = 0
                elif mag_id == 'Non-magnetic' and mag == 'FM':
                    mag_match = 1
                else:
                    mag_match = 2
                return (sg != input_sg_symbol, mag_match, dens_diff)

            sorted_ids = sorted(list(matches.keys()), key=sort_criteria)
            mp_id = sorted_ids[0]
            if mp_id in docs_by_mp_id:
                docs_by_mp_id[mp_id].append(doc)
            else:
                docs_by_mp_id[mp_id] = [doc]
        else:
            logger.debug("No material match found for formula {}".format(
                structure.composition.reduced_formula))
    return docs_by_mp_id
コード例 #27
0
ファイル: parse_outputs.py プロジェクト: lingtikong/atomate
    def run_task(self, fw_spec):
        # get the directory that contains the VASP dir to parse
        calc_dir = os.getcwd()
        if "calc_dir" in self:
            calc_dir = self["calc_dir"]
        elif self.get("calc_loc"):
            calc_dir = get_calc_loc(self["calc_loc"],
                                    fw_spec["calc_locs"])["path"]

        # parse the VASP directory
        logger.info("PARSING DIRECTORY: {}".format(calc_dir))

        drone = VaspDrone(additional_fields=self.get("additional_fields"),
                          parse_dos=self.get("parse_dos", False),
                          bandstructure_mode=self.get("bandstructure_mode",
                                                      False),
                          parse_chgcar=self.get("parse_chgcar", False),
                          parse_aeccar=self.get("parse_aeccar", False))

        # assimilate (i.e., parse)
        task_doc = drone.assimilate(calc_dir)

        # Check for additional keys to set based on the fw_spec
        if self.get("fw_spec_field"):
            task_doc.update(fw_spec[self.get("fw_spec_field")])

        # get the database connection
        db_file = env_chk(self.get('db_file'), fw_spec)

        # db insertion or taskdoc dump
        if not db_file:
            with open("task.json", "w") as f:
                f.write(json.dumps(task_doc, default=DATETIME_HANDLER))
        else:
            mmdb = VaspCalcDb.from_db_file(db_file, admin=True)
            t_id = mmdb.insert_task(
                task_doc,
                use_gridfs=self.get("parse_dos", False)
                or bool(self.get("bandstructure_mode", False))
                or self.get("parse_chgcar", False)
                or self.get("parse_aeccar", False))
            logger.info("Finished parsing with task_id: {}".format(t_id))

        defuse_children = False
        if task_doc["state"] != "successful":
            defuse_unsuccessful = self.get("defuse_unsuccessful",
                                           DEFUSE_UNSUCCESSFUL)
            if defuse_unsuccessful is True:
                defuse_children = True
            elif defuse_unsuccessful is False:
                pass
            elif defuse_unsuccessful == "fizzle":
                raise RuntimeError(
                    "VaspToDb indicates that job is not successful "
                    "(perhaps your job did not converge within the "
                    "limit of electronic/ionic iterations)!")
            else:
                raise RuntimeError("Unknown option for defuse_unsuccessful: "
                                   "{}".format(defuse_unsuccessful))

        task_fields_to_push = self.get("task_fields_to_push", None)
        update_spec = {}
        if task_fields_to_push:
            if isinstance(task_fields_to_push, dict):
                for key, path_in_task_doc in task_fields_to_push.items():
                    if has(task_doc, path_in_task_doc):
                        update_spec[key] = get(task_doc, path_in_task_doc)
                    else:
                        logger.warn(
                            "Could not find {} in task document. Unable to push to next firetask/firework"
                            .format(path_in_task_doc))
            else:
                raise RuntimeError(
                    "Inappropriate type {} for task_fields_to_push. It must be a "
                    "dictionary of format: {key: path} where key refers to a field "
                    "in the spec and path is a full mongo-style path to a "
                    "field in the task document".format(
                        type(task_fields_to_push)))

        return FWAction(stored_data={"task_id": task_doc.get("task_id", None)},
                        defuse_children=defuse_children,
                        update_spec=update_spec)
コード例 #28
0
 def handle_columns_reduce(total, column):
     value_ = get(item, column.order_by) if get(
         item, column.label) is None else get(item, column.label)
     total.append(value_)
     return total
コード例 #29
0
def form_response_new(formId):
    """
    Payload: 
    {
        "data": formData,
        "modifyLink": "...",
        "responseId"?: "asdsadasd"
    }
    If responseId is defined, it is an update.
    Otherwise, it is an existing submission.
    """
    from ..main import app
    email_sent = False
    responseId = app.current_request.json_body.get("responseId", None)
    if not responseId:
        responseId = ObjectId()
        newResponse = True
    else:
        responseId = ObjectId(responseId)
        newResponse = False

    form = Form.objects.get({"_id": ObjectId(formId)})
    response_data = app.current_request.json_body["data"]
    response_data = process_response_data_images(response_data)
    postprocess = form.formOptions.postprocess
    if postprocess and "patches" in postprocess and type(
            postprocess["patches"]) is list:
        response_data = patch_predicate(response_data, postprocess["patches"])
    counter_value = None
    counter = form.formOptions.counter
    if newResponse and counter and "enabled" in counter and counter[
            "enabled"] == True:
        counter_value = get_counter(formId)
    modify_link = app.current_request.json_body.get('modifyLink', '')
    paymentInfo = form.formOptions.paymentInfo
    confirmationEmailInfo = form.formOptions.confirmationEmailInfo
    paymentMethods = fill_paymentMethods_with_data(
        form.formOptions.paymentMethods, response_data)

    def calc_item_total_to_paymentInfo(paymentInfoItem, paymentInfo):
        paymentInfoItem['amount'] = calculate_price(
            paymentInfoItem.get('amount', '0'), response_data)
        paymentInfoItem['quantity'] = calculate_price(
            paymentInfoItem.get('quantity', '0'), response_data)
        paymentInfo[
            'total'] += paymentInfoItem['amount'] * paymentInfoItem['quantity']
        if "couponCode" in paymentInfoItem and paymentInfoItem[
                "amount"] * paymentInfoItem["quantity"] != 0:
            slots_maximum = calculate_price(
                paymentInfoItem.get("couponCodeMaximum", "-1"), response_data)
            if slots_maximum != -1:
                slots_requested = calculate_price(
                    paymentInfoItem.get("couponCodeCount", "1"), response_data)
                slots_used = form.couponCodes_used.get(
                    paymentInfoItem["couponCode"], 0)
                slots_available = slots_maximum - slots_used
                slots_remaining = slots_available - slots_requested
                if slots_remaining < 0:
                    message = "Coupon code maximum reached.\nSubmitting this form will cause you to exceed the coupon code maximum.\nNumber of spots remaining: {}".format(
                        int(slots_available))
                    return False, {
                        "res": {
                            "success": False,
                            "message": message,
                            "fields_to_clear": ["couponCode"]
                        }
                    }
                form.couponCodes_used[paymentInfoItem[
                    "couponCode"]] = slots_used + slots_requested
                Form.objects.raw({
                    "_id": form.id
                }).update({
                    "$set": {
                        f"couponCodes_used.{paymentInfoItem['couponCode']}":
                        slots_used + slots_requested
                    }
                })
        return True, {}

    paymentInfoItemsWithTotal = []
    paymentInfoItemsInstallment = []
    paymentInfo['total'] = 0
    for paymentInfoItem in paymentInfo.setdefault('items', []):
        paymentInfoItem.setdefault("name", "Payment Item")
        paymentInfoItem.setdefault("description", "Payment Item")
        paymentInfoItem.setdefault("quantity", "1")
        if paymentInfoItem.get("installment", False) == True:
            # Don't count "installment" payments towards the total.
            paymentInfoItemsInstallment.append(paymentInfoItem)
            continue
        if "$total" in paymentInfoItem.get(
                "amount", "0") or "$total" in paymentInfoItem.get(
                    "quantity", "0"):
            # Take care of this at the end.
            paymentInfoItemsWithTotal.append(paymentInfoItem)
            continue
        success, error = calc_item_total_to_paymentInfo(
            paymentInfoItem, paymentInfo)
        if success is False:
            return error

    # Now take care of items for round off, etc. -- which need the total value to work.
    response_data["total"] = float(paymentInfo["total"])
    for paymentInfoItem in paymentInfoItemsWithTotal:
        success, error = calc_item_total_to_paymentInfo(
            paymentInfoItem, paymentInfo)
        if success is False:
            return error

    # Take care of installment payments now.
    response_data["total"] = float(paymentInfo["total"])
    for paymentInfoItem in paymentInfoItemsInstallment:
        paymentInfoItem['amount'] = calculate_price(
            paymentInfoItem.get('amount', '0'), response_data)
        paymentInfoItem['quantity'] = calculate_price(
            paymentInfoItem.get('quantity', '0'), response_data)

    response_data.pop("total", None)

    paymentInfo['items'] = [
        item for item in paymentInfo['items']
        if item['quantity'] * item['amount'] != 0
    ]
    userId = app.get_current_user_id()
    paid = paymentInfo["total"] == 0
    if newResponse:
        response = Response(form=form,
                            id=responseId,
                            date_created=datetime.datetime.now(),
                            modify_link=modify_link + "?responseId=" +
                            str(responseId) if modify_link else "",
                            counter=counter_value)
        if get(form, "formOptions.loginRequired", False
               ) is True and userId is not "cm:cognitoUserPool:anonymousUser":
            user = get_user_or_create_one(userId)
            response.user = userId
            # Only one response per user.
            try:
                Response.objects.get({
                    "form": ObjectId(formId),
                    "user": userId
                })
                raise Exception(
                    f"Response with userId {userId} already exists!")
            except DoesNotExist:
                pass
    else:
        response = Response.objects.get({"_id": responseId})
        response.update_trail.append(
            UpdateTrailItem(old=response.value,
                            new=response_data,
                            date=datetime.datetime.now(),
                            update_type="update"))
        if (response.paid == True
                and paymentInfo["total"] <= response.paymentInfo["total"]):
            paid = True
        if form.id != response.form.id:
            raise UnauthorizedError(
                f"Response {response.id} does not belong to form {form.id}; it belongs to form {response.form.id}."
            )
        if response.user and response.user.id != userId:
            app.check_permissions(form, 'Responses_Edit')
            # raise UnauthorizedError(f"User {userId} does not own response {response.id} (owner is {response.user.id})")
    if newResponse or (not newResponse and paid):
        response.value = response_data
        response.date_modified = datetime.datetime.now()
        response.paymentInfo = paymentInfo
        response.paid = paid
        if not newResponse:
            response.update_trail.append(
                UpdateTrailItem(date=datetime.datetime.now(),
                                update_type="apply_update"))
        if paid and confirmationEmailInfo:  # If total amount is zero (user uses coupon code to get for free)
            send_confirmation_email(response, confirmationEmailInfo)
            email_sent = True
        # todo: fix this, should auto_email come even if not paid?
        if "auto_email" in paymentMethods and get(
                paymentMethods, "auto_email.enabled", True) == True and type(
                    get(paymentMethods, "autoEmail.confirmationEmailInfo") is
                    dict):
            send_confirmation_email(
                response,
                get(paymentMethods, "auto_email.confirmationEmailInfo"))
            email_sent = True
        response.save()
        if "description" in paymentInfo and type(
                paymentInfo["description"]) is str:
            paymentInfo["description"] = fill_string_from_template(
                response, paymentInfo["description"])
        if "ccavenue" in paymentMethods and response.paid == False:
            paymentMethods["ccavenue"] = update_ccavenue_hash(
                formId, paymentMethods["ccavenue"], response)
        return {
            "res": {
                "value": response_data,
                "paid": paid,
                "success": True,
                "action": "insert",
                "email_sent": email_sent,
                "responseId": str(responseId),
                "paymentInfo": paymentInfo,
                "paymentMethods": paymentMethods
            }
        }
    elif not newResponse:
        # Update.
        response.date_modified = datetime.datetime.now()
        # Not using pending_update for now.
        # response.pending_update = {
        #     "value": response_data,
        #     "paymentInfo": paymentInfo,
        # }
        response.value = response_data
        response.paymentInfo = paymentInfo
        response.paid = paid
        response.save()
        if "description" in paymentInfo and type(
                paymentInfo["description"]) is str:
            paymentInfo["description"] = fill_string_from_template(
                response, paymentInfo["description"])
        if "ccavenue" in paymentMethods and response.paid == False:
            paymentMethods["ccavenue"] = update_ccavenue_hash(
                formId, paymentMethods["ccavenue"], response)
        return {
            "res": {
                "value": response_data,
                "paid": paid,
                "success": True,
                "action": "update",
                "email_sent": email_sent,
                "responseId": str(responseId),
                "paymentInfo": paymentInfo,
                "paymentMethods": paymentMethods,
                "amt_received": {
                    "currency": paymentInfo["currency"],
                    "total": float(response.amount_paid or 0)
                }
            }
        }
コード例 #30
0
def get(path):
    return objects.get(_config, path)
コード例 #31
0
    def active_version(self, data, **kwargs):
        from implement_table.core.classes import BConfig
        reference_id = data.pop(self.reference_field)

        date_ = get(data, 'max_date', None)

        query = {
            self.reference_field: int(reference_id),
        }

        if date_ is not None:
            query[self.date_field] = {
                "$lt":
                datetime.strptime(date_,
                                  BConfig().date_format) + timedelta(days=1)
            }

        subscriber = kwargs.get('subscriber')
        query.update(
            self.permission_filter(subscriber.user, subscriber.staff, 'get'))

        sort = [{
            '$match': query
        }, {
            "$group": {
                "_id": {
                    "$dateToString": {
                        "format": "%Y-%m-%d",
                        "date": "${}".format(self.date_field)
                    }
                },
                "pk": {
                    "$last": "$_id"
                },
                "created_at": {
                    "$last": "${}".format(self.date_field)
                }
            }
        }, {
            '$sort': {
                self.date_field: -1
            }
        }, {
            '$limit': 1
        }, {
            "$project": {
                "_id": "$pk"
            }
        }]

        object__ = self.get_model().objects.aggregate(*sort)

        if len(object__) > 0:
            pk = get(object__, '0._id')
            object__ = self.get_model().objects.get(id=pk)

            serializer = self.get_serializer(object__)

            return serializer.data
        else:
            return dict()
コード例 #32
0
def _search(form, query, autocomplete, search_by_id, show_unpaid):
    search_fields = get(form.formOptions.dataOptions, "search.searchFields", ["_id"])
    if search_by_id is not None:
        search_fields = ["_id"]
    result_limit = get(form.formOptions.dataOptions, "search.resultLimit", 10)
    result_fields = get(form.formOptions.dataOptions, "search.resultFields", ["_id"])
    autocomplete_fields = get(
        form.formOptions.dataOptions, "search.autocompleteFields", ["_id"]
    )
    if show_unpaid is not None:
        default_mongo_query = {"paid": False}
    else:
        default_mongo_query = {"paid": True}
    mongo_query = {"$or": []}
    for word in query.split(" "):
        for field in search_fields:
            if field == "_id":
                if len(word) <= 24:
                    try:
                        queryObjectIdStart = ObjectId(
                            word + "0" * (24 - len(word))
                        )  # fill in zeroes to create object id, e.g. 5cba --> 5cba0000000000000000000
                        queryObjectIdEnd = ObjectId(word + "e" * (24 - len(word)))
                        mongo_query["$or"].append(
                            {
                                field: {
                                    "$gte": queryObjectIdStart,
                                    "$lte": queryObjectIdEnd,
                                }
                            }
                        )
                    except bson.errors.InvalidId:
                        pass
            else:
                if field.startswith("value.participants."):
                    _, subfield = field.split("value.participants.")
                    mongo_query["$or"].append(
                        {
                            "value.participants": {
                                "$elemMatch": {
                                    subfield: {"$regex": "^" + word, "$options": "i"}
                                }
                            }
                        }
                    )
                else:
                    mongo_query["$or"].append(
                        {field: {"$regex": "^" + word, "$options": "i"}}
                    )
    mongo_query["form"] = form.id
    if len(mongo_query["$or"]) == 0:
        del mongo_query["$or"]
    # Default query paid = True
    if mongo_query:
        mongo_query = {"$and": [default_mongo_query, mongo_query]}
    else:
        mongo_query = default_mongo_query
    if autocomplete is not None:
        projection = {field: 1 for field in autocomplete_fields}
        result_limit = 5
    else:
        projection = {}
        for field in result_fields:
            projection[field] = 1
    responses = (
        Response.objects.raw(mongo_query).limit(result_limit).project(projection)
    )
    return {"res": [serialize_model(r) for r in responses]}