def test_1(self):
self.maxDiff = None
data_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_path, "./data/test_globals.xml")
json_ref_path = os.path.join(data_path, "./data/test_globals_1.json")
table_mapper = TableMapper("Results"
, votable_path
, json_inst_dict=DictUtils.read_dict_from_file(json_ref_path))
table_mapper.resolve_refs_and_values(resolve_refs=True)
#print(DictUtils.get_pretty_json(table_mapper.json["MODEL_INSTANCE"]["TABLE_MAPPING"]["my:other.role"]))
self.assertDictEqual(table_mapper.json["MODEL_INSTANCE"]["TABLE_MAPPING"]["Results"]["my:other.role"]
,{
"@ID": "TestParamRef",
"@dmtype": "Whatever",
"coords:whatever": {
"@dmtype": "coords:StdRefLocation",
"coords:StdRefLocation.position": {
"@dmtype": "ivoa:string",
"@ref": "param_ref",
"@value": "param_value"
}
}
}
, "")
def test_2(self):
self.maxDiff = None
data_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_path,
"./data/test_multi_templates.xml")
json_ref_path = os.path.join(data_path,
"./data/test_multi_templates.json")
logger.info("extract vodml block from %s", votable_path)
instanceFromVotable = InstanceFromVotable(votable_path)
instanceFromVotable._extract_vodml_block()
instanceFromVotable._validate_vodml_block()
builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block)
builder.revert_compositions("COLLECTION")
builder.revert_templates()
builder.revert_elements("INSTANCE")
builder.revert_elements("ATTRIBUTE")
#print(DictUtils.get_pretty_json(builder.json))
self.assertDictEqual(
builder.json["MODEL_INSTANCE"],
DictUtils.read_dict_from_file(json_ref_path)["MODEL_INSTANCE"],
"=======")
def _revert_set_2bethrown(self, root_element, name, default_key):
print("aaaa")
if isinstance(root_element, list):
for idx, _ in enumerate(root_element):
if self.retour is None:
self._revert_set(root_element[idx], name, default_key)
elif isinstance(root_element, dict):
for k, v in root_element.items():
if k == name:
if isinstance(v, list):
raise Exception(name + " cannot be a list")
elif isinstance(v, dict):
# the elememt may have been reverted in a precedent pass
# It can not be reserved twice
try:
print("bbbbbbbbbbb ")
print(DictUtils.get_pretty_json(v))
newcontent = {}
new_key = self._get_key_for_element(v["INSTANCE"], default_key)
newcontent[new_key] = deepcopy(v["INSTANCE"])
self.retour = {'node': root_element, "newcontent": newcontent}
except:
import traceback
traceback.print_exc()
pass
if self.retour is None:
self._revert_set(v, name, default_key)
def test_1(self):
self.maxDiff = None
data_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_path, "./data/test_filter.xml")
json_ref_path = os.path.join(data_path, "./data/test_filter_1.json")
table_mapper = TableMapper("Results"
, votable_path
, json_inst_dict=DictUtils.read_dict_from_file(json_ref_path))
table_mapper.resolve_refs_and_values()
self.assertListEqual([*table_mapper.table_iterators], ['meas:Error.statError'], "")
self.assertDictEqual(table_mapper.table_iterators['meas:Error.statError'].column_mapping.column_refs
, {'_poserr_148': {'parent_role': 'meas:Symmetrical.radius', 'role': 'ivoa:RealQuantity.value', 'index': None, 'field': None}}
, "")
table_mapper.map_columns()
self.assertDictEqual(table_mapper.table_iterators['meas:Error.statError'].column_mapping.column_ids
, {0: {'name': 'oidsaada', 'ref': None, 'id': '_poserr_148'}}
, "")
self.assertEqual(table_mapper.table_iterators['meas:Error.statError'].row_filter.__repr__()
, "Row filter NoName key=_poserr_148 value=2 col=-1"
, "")
self.assertListEqual(table_mapper.get_flatten_data_head()
,['meas:Symmetrical.radius(ivoa:RealQuantity.value) [col#0 _poserr_148]'],""),
cpt=2
while True:
inst = table_mapper._get_next_flatten_row()
if inst != None:
self.assertListEqual(inst, [cpt], "")
cpt += 1
else:
break
table_mapper.rewind()
cpt=2
while True:
inst = table_mapper._get_next_row_instance()
if inst != None:
self.assertDictEqual(inst
, {
"@dmtype": "meas:Symmetrical",
"meas:Symmetrical.radius": {
"@dmtype": "ivoa:RealQuantity",
"ivoa:Quantity.unit": {
"@dmtype": "ivoa:Unit",
"@value": "arcsec"
},
"ivoa:RealQuantity.value": {
"@dmtype": "ivoa:real",
"@ref": "_poserr_148",
"@value": cpt
}
}
}
, "")
cpt+=1
else:
break
def test_22(self):
if self.populated is False:
self.vodml_instance.populate_templates()
self.vodml_instance.connect_join_iterators()
self.populated = True
table_mapper = self.vodml_instance.table_mappers["Results"]
full_dict = table_mapper.get_full_instance()
#
#print(DictUtils.get_pretty_json(full_dict))
self.assertDictEqual(full_dict, DictUtils.read_dict_from_file(self.json_ref_path), "")
def test_1(self):
self.json_ref_path = os.path.join(
self.data_path, "./data/test_vodml_instance_21.json")
if self.populated is False:
self.vodml_instance.populate_templates()
self.vodml_instance.connect_join_iterators()
self.populated = True
table_mapper = self.vodml_instance.table_mappers["Results"]
full_dict = table_mapper.get_full_instance()
print(DictUtils.get_pretty_json(full_dict))
sys.exit(1)
def test_2(self):
self.json_ref_path = os.path.join(
self.data_path, "./data/test_inst_fullfeature_2.json")
self.maxDiff = None
self.table_mapper.resolve_refs_and_values()
self.table_mapper.map_columns()
full_dict = self.table_mapper.get_full_instance(resolve_refs=True)
#print(DictUtils.get_pretty_json(full_dict))
#print( DictUtils.read_dict_from_file(json_ref_path))
self.assertDictEqual(full_dict,
DictUtils.read_dict_from_file(self.json_ref_path),
"")
def get(self, params):
if self.get_argument("fetch", True):
scope = ['https://spreadsheets.google.com/feeds']
credentials = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json',scope)
word_repository = WordRepository()
dict_utils = DictUtils()
try:
gc = gspread.authorize(credentials)
sh = gc.open_by_key("1QeU3AoSghCAvYBD8kauC8oEZ0wA6j_gPj1pkvIY4MPU")
for ws_count, worksheets in enumerate(sh.worksheets()):
word_list = worksheets.col_values(1)
type_list = worksheets.col_values(2)
meaning_list = worksheets.col_values(3)
if ws_count < 26:
word_tuple_list = []
for i, (word, word_type, meaning) in enumerate(zip(word_list, type_list, meaning_list)):
if i > 0:
if word != '':
word_tuple_list.append(dict(id=str(uuid.uuid1().hex), word=word,
type=word_type,
meaning_zg=meaning,
meaning_uni=meaning))
else:
pass
word_repository.bulk_insert(word_tuple_list, dict_utils.get_model(str(worksheets.title).lower()))
else:
self.respond({}, NO_CONTENT_ERROR, code=SUCCESS)
except Exception as ex:
print ex.message
self.respond({}, ex.message, code=SERVER_ERROR)
def test_21(self):
self.json_ref_path = os.path.join(
self.data_path, "./data/test_vodml_instance_21.json")
if self.populated is False:
self.vodml_instance.populate_templates()
self.vodml_instance.connect_join_iterators()
self.populated = True
table_mapper = self.vodml_instance.table_mappers["OtherResults"]
full_dict = table_mapper.get_full_instance()
#print(DictUtils.get_pretty_json(full_dict))
self.assertDictEqual(full_dict,
DictUtils.read_dict_from_file(self.json_ref_path),
"")
def test_1(self):
votable_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(votable_path, "./data/test_instance.xml")
json_ref_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./data/test_instance_1.json")
logger.info("extract vodml block from %s", votable_path)
instanceFromVotable = InstanceFromVotable(votable_path)
instanceFromVotable._extract_vodml_block()
instanceFromVotable._validate_vodml_block()
builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block)
builder.revert_elements("INSTANCE")
builder.revert_elements("ATTRIBUTE")
#print(DictUtils.get_pretty_json(builder.json["VODML"]["TEMPLATES"]))
self.assertDictEqual(builder.json["VODML"]["TEMPLATES"], DictUtils.read_dict_from_file(json_ref_path), "=======")
def test_1(self):
data_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_path, "./data/test_composition.xml")
json_ref_path = os.path.join(data_path,
"./data/test_composition_1.json")
logger.info("extract vodml block from %s", votable_path)
instanceFromVotable = InstanceFromVotable(votable_path)
instanceFromVotable._extract_vodml_block()
instanceFromVotable._validate_vodml_block()
builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block)
builder.revert_compositions("COLLECTION")
#print(DictUtils.get_pretty_json(builder.json["MODEL_INSTANCE"]["TABLE_MAPPING"]))
self.assertDictEqual(builder.json["MODEL_INSTANCE"]["TABLE_MAPPING"],
DictUtils.read_dict_from_file(json_ref_path),
"=======")
def _build_instance(self):
'''
Translate the MODEL_INSTANCE block into dict
'''
builder = JsonMappingBuilder(json_dict=self.json_block)
builder.revert_sets("GLOBALS", default_key='globals')
#self.builder.revert_compositions("COLLECTION")
builder.revert_sets("TABLE_MAPPING", default_key='root')
builder.revert_array()
builder.revert_compositions("COLLECTION")
builder.revert_elements("INSTANCE")
builder.revert_elements("ATTRIBUTE")
builder.revert_elements("MODEL")
self.json_vodml_block = builder.json
print(DictUtils.get_pretty_json(self.json_vodml_block))
logger.info("JSON MODEL_INSTANCE block built")
def test_1(self):
self.maxDiff = None
data_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_path, "./data/test_filter.xml")
json_ref_path = os.path.join(data_path, "./data/test_filter_1.json")
logger.info("extract vodml block from %s", votable_path)
instanceFromVotable = InstanceFromVotable(votable_path)
instanceFromVotable._extract_vodml_block()
instanceFromVotable._validate_vodml_block()
builder = JsonMappingBuilder(json_dict=instanceFromVotable.json_block)
#builder.revert_array()
builder.revert_compositions("COLLECTION")
builder.revert_templates()
builder.revert_elements("INSTANCE")
builder.revert_elements("ATTRIBUTE")
self.assertDictEqual(json.loads(json.dumps(builder.json)),
DictUtils.read_dict_from_file(json_ref_path),
"=======")
from utils.dict_utils import DictUtils
from client.inst_builder.vodml_instance import VodmlInstance
if __name__ == '__main__':
base_path = os.path.dirname(os.path.realpath(__file__))
votable_path = os.path.join(data_dir, "annotated_data",
"vizier_votable_avecActivity_Vo-dml-lite.xml")
vodml_instance = VodmlInstance(votable_path)
vodml_instance.populate_templates()
vodml_instance.connect_join_iterators()
instance = vodml_instance.get_root_element("voprov:Entity")
print(DictUtils.get_pretty_json(instance.json))
print("=== Mapping of the columns")
print(instance.get_flatten_data_head())
#print(instance.get_data_subset_keys())
print("=== First row: flatten mode")
while True:
inst = instance._get_next_flatten_row()
if inst != None:
print(DictUtils.get_pretty_json(inst))
break
else:
break
print("=== Second row: instance mode")
while True:
inst = instance._get_next_row_instance()
parser.add_argument('--drop_out', type=float, default=0.5)
parser.add_argument('--m', type=float, default=0.3)
parser.add_argument('--p', type=float, default=0.55)
parser.add_argument('--flag', default="PER")
parser.add_argument('--dataset', default="conll2003")
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--batch_size', type=int, default=300)
parser.add_argument('--model', default="")
parser.add_argument('--iter', type=int, default=1)
args = parser.parse_args()
dp = DataPrepare(args.dataset)
mutils = AdaptivePUUtils(dp)
dutils = DictUtils()
trainSet, validSet, testSet, prior = mutils.load_new_dataset(
args.flag, args.dataset, args.iter, args.p)
print(prior)
trainSize = len(trainSet)
validSize = len(validSet)
testSize = len(testSet)
print(("train set size: {}, valid set size: {}, test set size: {}").format(
trainSize, validSize, testSize))
charcnn = CharCNN(dp.char2Idx)
wordnet = WordNet(dp.wordEmbeddings, dp.word2Idx)
casenet = CaseNet(dp.caseEmbeddings, dp.case2Idx)
featurenet = FeatureNet()
pulstmcnn = AdaPULSTMCNN2(dp, charcnn, wordnet, casenet, featurenet, 150,
def test_1(self):
#print(DictUtils.get_pretty_json(self.vodml_instance.json_view))
self.assertDictEqual(self.vodml_instance.json_view,
DictUtils.read_dict_from_file(self.json_ref_path),
"=======")
if inst != None:
break
else:
break
#BUG 2 parameters arrays are nested
#TODO 2 be fixed
parameters = inst["mango:MangoObject.parameters"]
parameters = parameters[0]["mango:MangoObject.parameters"]
parameter_map = {}
for parameter in parameters:
ucd = parameter["mango:Parameter.ucd"]["@value"]
classe = parameter["mango:Parameter.measure"]["@dmtype"]
semantic = parameter["mango:Parameter.semantic"]["@value"]
description = parameter["mango:Parameter.description"]["@value"]
parameter_map[classe] = {
"class": classe,
"ucd": ucd,
"semantic": semantic,
"description": description
}
return parameter_map
if __name__ == '__main__':
base_path = os.path.dirname(os.path.realpath(__file__))
for vot in ["vizier_propermotion.annot.xml", "4xmm_detections.annot.xml"]:
votable_path = os.path.join(data_dir, "annotated_data", vot)
parameter_map = map_measures(votable_path)
print(DictUtils.get_pretty_json(parameter_map))
def save_instance(self):
file_path = self.json_path.replace(".json", ".inst.json")
logger.info("save instance in %s", file_path)
with open(file_path, 'w') as jsonfile:
jsonfile.write(DictUtils.get_pretty_json(self.json))
vodml_instance.connect_join_iterators()
table_row_instances = vodml_instance.get_root_element("mango:MangoObject")
if len(vodml_instance.table_mappers) == 0:
print("no table mapper")
sys.exit(1)
mapper_name = None
for k, v in vodml_instance.table_mappers.items():
print("process table mapper {}".format(k))
mapper_name = k
break;
while True:
inst = table_row_instances._get_next_row_instance()
if inst != None:
print(DictUtils.get_pretty_json(inst))
break
wrapper = AstropyWrapper(vodml_instance, mapper_name)
print("Astropy space frame: {}".format(wrapper.get_space_frame(inst)))
print("Astropy time frame: {}".format(wrapper.get_time_frame(inst)))
