def test_1(self): self.maxDiff = None data_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_path, "./data/test_globals.xml") json_ref_path = os.path.join(data_path, "./data/test_globals_1.json") table_mapper = TableMapper("Results" , votable_path , json_inst_dict=DictUtils.read_dict_from_file(json_ref_path)) table_mapper.resolve_refs_and_values(resolve_refs=True) #print(DictUtils.get_pretty_json(table_mapper.json["MODEL_INSTANCE"]["TABLE_MAPPING"]["my:other.role"])) self.assertDictEqual(table_mapper.json["MODEL_INSTANCE"]["TABLE_MAPPING"]["Results"]["my:other.role"] ,{ "@ID": "TestParamRef", "@dmtype": "Whatever", "coords:whatever": { "@dmtype": "coords:StdRefLocation", "coords:StdRefLocation.position": { "@dmtype": "ivoa:string", "@ref": "param_ref", "@value": "param_value" } } } , "")
def test_2(self): self.maxDiff = None data_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_path, "./data/test_multi_templates.xml") json_ref_path = os.path.join(data_path, "./data/test_multi_templates.json") logger.info("extract vodml block from %s", votable_path) instanceFromVotable = InstanceFromVotable(votable_path) instanceFromVotable._extract_vodml_block() instanceFromVotable._validate_vodml_block() builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block) builder.revert_compositions("COLLECTION") builder.revert_templates() builder.revert_elements("INSTANCE") builder.revert_elements("ATTRIBUTE") #print(DictUtils.get_pretty_json(builder.json)) self.assertDictEqual( builder.json["MODEL_INSTANCE"], DictUtils.read_dict_from_file(json_ref_path)["MODEL_INSTANCE"], "=======")
def _revert_set_2bethrown(self, root_element, name, default_key): print("aaaa") if isinstance(root_element, list): for idx, _ in enumerate(root_element): if self.retour is None: self._revert_set(root_element[idx], name, default_key) elif isinstance(root_element, dict): for k, v in root_element.items(): if k == name: if isinstance(v, list): raise Exception(name + " cannot be a list") elif isinstance(v, dict): # the elememt may have been reverted in a precedent pass # It can not be reserved twice try: print("bbbbbbbbbbb ") print(DictUtils.get_pretty_json(v)) newcontent = {} new_key = self._get_key_for_element(v["INSTANCE"], default_key) newcontent[new_key] = deepcopy(v["INSTANCE"]) self.retour = {'node': root_element, "newcontent": newcontent} except: import traceback traceback.print_exc() pass if self.retour is None: self._revert_set(v, name, default_key)
def test_1(self): self.maxDiff = None data_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_path, "./data/test_filter.xml") json_ref_path = os.path.join(data_path, "./data/test_filter_1.json") table_mapper = TableMapper("Results" , votable_path , json_inst_dict=DictUtils.read_dict_from_file(json_ref_path)) table_mapper.resolve_refs_and_values() self.assertListEqual([*table_mapper.table_iterators], ['meas:Error.statError'], "") self.assertDictEqual(table_mapper.table_iterators['meas:Error.statError'].column_mapping.column_refs , {'_poserr_148': {'parent_role': 'meas:Symmetrical.radius', 'role': 'ivoa:RealQuantity.value', 'index': None, 'field': None}} , "") table_mapper.map_columns() self.assertDictEqual(table_mapper.table_iterators['meas:Error.statError'].column_mapping.column_ids , {0: {'name': 'oidsaada', 'ref': None, 'id': '_poserr_148'}} , "") self.assertEqual(table_mapper.table_iterators['meas:Error.statError'].row_filter.__repr__() , "Row filter NoName key=_poserr_148 value=2 col=-1" , "") self.assertListEqual(table_mapper.get_flatten_data_head() ,['meas:Symmetrical.radius(ivoa:RealQuantity.value) [col#0 _poserr_148]'],""), cpt=2 while True: inst = table_mapper._get_next_flatten_row() if inst != None: self.assertListEqual(inst, [cpt], "") cpt += 1 else: break table_mapper.rewind() cpt=2 while True: inst = table_mapper._get_next_row_instance() if inst != None: self.assertDictEqual(inst , { "@dmtype": "meas:Symmetrical", "meas:Symmetrical.radius": { "@dmtype": "ivoa:RealQuantity", "ivoa:Quantity.unit": { "@dmtype": "ivoa:Unit", "@value": "arcsec" }, "ivoa:RealQuantity.value": { "@dmtype": "ivoa:real", "@ref": "_poserr_148", "@value": cpt } } } , "") cpt+=1 else: break
def test_22(self): if self.populated is False: self.vodml_instance.populate_templates() self.vodml_instance.connect_join_iterators() self.populated = True table_mapper = self.vodml_instance.table_mappers["Results"] full_dict = table_mapper.get_full_instance() # #print(DictUtils.get_pretty_json(full_dict)) self.assertDictEqual(full_dict, DictUtils.read_dict_from_file(self.json_ref_path), "")
def test_1(self): self.json_ref_path = os.path.join( self.data_path, "./data/test_vodml_instance_21.json") if self.populated is False: self.vodml_instance.populate_templates() self.vodml_instance.connect_join_iterators() self.populated = True table_mapper = self.vodml_instance.table_mappers["Results"] full_dict = table_mapper.get_full_instance() print(DictUtils.get_pretty_json(full_dict)) sys.exit(1)
def test_2(self): self.json_ref_path = os.path.join( self.data_path, "./data/test_inst_fullfeature_2.json") self.maxDiff = None self.table_mapper.resolve_refs_and_values() self.table_mapper.map_columns() full_dict = self.table_mapper.get_full_instance(resolve_refs=True) #print(DictUtils.get_pretty_json(full_dict)) #print( DictUtils.read_dict_from_file(json_ref_path)) self.assertDictEqual(full_dict, DictUtils.read_dict_from_file(self.json_ref_path), "")
def get(self, params): if self.get_argument("fetch", True): scope = ['https://spreadsheets.google.com/feeds'] credentials = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json',scope) word_repository = WordRepository() dict_utils = DictUtils() try: gc = gspread.authorize(credentials) sh = gc.open_by_key("1QeU3AoSghCAvYBD8kauC8oEZ0wA6j_gPj1pkvIY4MPU") for ws_count, worksheets in enumerate(sh.worksheets()): word_list = worksheets.col_values(1) type_list = worksheets.col_values(2) meaning_list = worksheets.col_values(3) if ws_count < 26: word_tuple_list = [] for i, (word, word_type, meaning) in enumerate(zip(word_list, type_list, meaning_list)): if i > 0: if word != '': word_tuple_list.append(dict(id=str(uuid.uuid1().hex), word=word, type=word_type, meaning_zg=meaning, meaning_uni=meaning)) else: pass word_repository.bulk_insert(word_tuple_list, dict_utils.get_model(str(worksheets.title).lower())) else: self.respond({}, NO_CONTENT_ERROR, code=SUCCESS) except Exception as ex: print ex.message self.respond({}, ex.message, code=SERVER_ERROR)
def test_21(self): self.json_ref_path = os.path.join( self.data_path, "./data/test_vodml_instance_21.json") if self.populated is False: self.vodml_instance.populate_templates() self.vodml_instance.connect_join_iterators() self.populated = True table_mapper = self.vodml_instance.table_mappers["OtherResults"] full_dict = table_mapper.get_full_instance() #print(DictUtils.get_pretty_json(full_dict)) self.assertDictEqual(full_dict, DictUtils.read_dict_from_file(self.json_ref_path), "")
def test_1(self): votable_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(votable_path, "./data/test_instance.xml") json_ref_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./data/test_instance_1.json") logger.info("extract vodml block from %s", votable_path) instanceFromVotable = InstanceFromVotable(votable_path) instanceFromVotable._extract_vodml_block() instanceFromVotable._validate_vodml_block() builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block) builder.revert_elements("INSTANCE") builder.revert_elements("ATTRIBUTE") #print(DictUtils.get_pretty_json(builder.json["VODML"]["TEMPLATES"])) self.assertDictEqual(builder.json["VODML"]["TEMPLATES"], DictUtils.read_dict_from_file(json_ref_path), "=======")
def test_1(self): data_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_path, "./data/test_composition.xml") json_ref_path = os.path.join(data_path, "./data/test_composition_1.json") logger.info("extract vodml block from %s", votable_path) instanceFromVotable = InstanceFromVotable(votable_path) instanceFromVotable._extract_vodml_block() instanceFromVotable._validate_vodml_block() builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block) builder.revert_compositions("COLLECTION") #print(DictUtils.get_pretty_json(builder.json["MODEL_INSTANCE"]["TABLE_MAPPING"])) self.assertDictEqual(builder.json["MODEL_INSTANCE"]["TABLE_MAPPING"], DictUtils.read_dict_from_file(json_ref_path), "=======")
def _build_instance(self): ''' Translate the MODEL_INSTANCE block into dict ''' builder = JsonMappingBuilder(json_dict=self.json_block) builder.revert_sets("GLOBALS", default_key='globals') #self.builder.revert_compositions("COLLECTION") builder.revert_sets("TABLE_MAPPING", default_key='root') builder.revert_array() builder.revert_compositions("COLLECTION") builder.revert_elements("INSTANCE") builder.revert_elements("ATTRIBUTE") builder.revert_elements("MODEL") self.json_vodml_block = builder.json print(DictUtils.get_pretty_json(self.json_vodml_block)) logger.info("JSON MODEL_INSTANCE block built")
def test_1(self): self.maxDiff = None data_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_path, "./data/test_filter.xml") json_ref_path = os.path.join(data_path, "./data/test_filter_1.json") logger.info("extract vodml block from %s", votable_path) instanceFromVotable = InstanceFromVotable(votable_path) instanceFromVotable._extract_vodml_block() instanceFromVotable._validate_vodml_block() builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block) #builder.revert_array() builder.revert_compositions("COLLECTION") builder.revert_templates() builder.revert_elements("INSTANCE") builder.revert_elements("ATTRIBUTE") self.assertDictEqual(json.loads(json.dumps(builder.json)), DictUtils.read_dict_from_file(json_ref_path), "=======")
from utils.dict_utils import DictUtils from client.inst_builder.vodml_instance import VodmlInstance if __name__ == '__main__': base_path = os.path.dirname(os.path.realpath(__file__)) votable_path = os.path.join(data_dir, "annotated_data", "vizier_votable_avecActivity_Vo-dml-lite.xml") vodml_instance = VodmlInstance(votable_path) vodml_instance.populate_templates() vodml_instance.connect_join_iterators() instance = vodml_instance.get_root_element("voprov:Entity") print(DictUtils.get_pretty_json(instance.json)) print("=== Mapping of the columns") print(instance.get_flatten_data_head()) #print(instance.get_data_subset_keys()) print("=== First row: flatten mode") while True: inst = instance._get_next_flatten_row() if inst != None: print(DictUtils.get_pretty_json(inst)) break else: break print("=== Second row: instance mode") while True: inst = instance._get_next_row_instance()
parser.add_argument('--drop_out', type=float, default=0.5) parser.add_argument('--m', type=float, default=0.3) parser.add_argument('--p', type=float, default=0.55) parser.add_argument('--flag', default="PER") parser.add_argument('--dataset', default="conll2003") parser.add_argument('--lr', type=float, default=1e-4) parser.add_argument('--batch_size', type=int, default=300) parser.add_argument('--model', default="") parser.add_argument('--iter', type=int, default=1) args = parser.parse_args() dp = DataPrepare(args.dataset) mutils = AdaptivePUUtils(dp) dutils = DictUtils() trainSet, validSet, testSet, prior = mutils.load_new_dataset( args.flag, args.dataset, args.iter, args.p) print(prior) trainSize = len(trainSet) validSize = len(validSet) testSize = len(testSet) print(("train set size: {}, valid set size: {}, test set size: {}").format( trainSize, validSize, testSize)) charcnn = CharCNN(dp.char2Idx) wordnet = WordNet(dp.wordEmbeddings, dp.word2Idx) casenet = CaseNet(dp.caseEmbeddings, dp.case2Idx) featurenet = FeatureNet() pulstmcnn = AdaPULSTMCNN2(dp, charcnn, wordnet, casenet, featurenet, 150,
def test_1(self): #print(DictUtils.get_pretty_json(self.vodml_instance.json_view)) self.assertDictEqual(self.vodml_instance.json_view, DictUtils.read_dict_from_file(self.json_ref_path), "=======")
if inst != None: break else: break #BUG 2 parameters arrays are nested #TODO 2 be fixed parameters = inst["mango:MangoObject.parameters"] parameters = parameters[0]["mango:MangoObject.parameters"] parameter_map = {} for parameter in parameters: ucd = parameter["mango:Parameter.ucd"]["@value"] classe = parameter["mango:Parameter.measure"]["@dmtype"] semantic = parameter["mango:Parameter.semantic"]["@value"] description = parameter["mango:Parameter.description"]["@value"] parameter_map[classe] = { "class": classe, "ucd": ucd, "semantic": semantic, "description": description } return parameter_map if __name__ == '__main__': base_path = os.path.dirname(os.path.realpath(__file__)) for vot in ["vizier_propermotion.annot.xml", "4xmm_detections.annot.xml"]: votable_path = os.path.join(data_dir, "annotated_data", vot) parameter_map = map_measures(votable_path) print(DictUtils.get_pretty_json(parameter_map))
def save_instance(self): file_path = self.json_path.replace(".json", ".inst.json") logger.info("save instance in %s", file_path) with open(file_path, 'w') as jsonfile: jsonfile.write(DictUtils.get_pretty_json(self.json))
vodml_instance.connect_join_iterators() table_row_instances = vodml_instance.get_root_element("mango:MangoObject") if len(vodml_instance.table_mappers) == 0: print("no table mapper") sys.exit(1) mapper_name = None for k, v in vodml_instance.table_mappers.items(): print("process table mapper {}".format(k)) mapper_name = k break; while True: inst = table_row_instances._get_next_row_instance() if inst != None: print(DictUtils.get_pretty_json(inst)) break wrapper = AstropyWrapper(vodml_instance, mapper_name) print("Astropy space frame: {}".format(wrapper.get_space_frame(inst))) print("Astropy time frame: {}".format(wrapper.get_time_frame(inst)))