def _load_data_from_sample(hyperparameters: Dict[str, Any],
metadata: Dict[str, Any],
raw_sample: Dict[str, Any],
result_holder: Dict[str, Any],
is_train: bool = True) -> bool:
keep_sample = super(Path2Annotation,
Path2Annotation)._load_data_from_sample(
hyperparameters, metadata, raw_sample,
result_holder, is_train)
if not keep_sample:
return False
target_class = []
for node_idx, annotation_data in raw_sample['supernodes'].items():
annotation = annotation_data['annotation']
if is_train and ignore_type_annotation(annotation):
continue
target_class.append(
TypeClassificationModel._get_idx_for_type(
annotation, metadata, hyperparameters))
result_holder['variable_target_class'] = np.array(target_class,
dtype=np.uint16)
return len(target_class) > 0
def _load_data_from_sample(hyperparameters: Dict[str, Any],
metadata: Dict[str, Any],
raw_sample: Dict[str, Any],
result_holder: Dict[str, Any],
is_train: bool = True) -> bool:
keep_sample = super(Graph2HybridMetric,
Graph2HybridMetric)._load_data_from_sample(
hyperparameters, metadata, raw_sample,
result_holder, is_train)
if not keep_sample:
return False
target_node_idxs, target_class, target_class_id = [], [], []
for node_idx, annotation_data in raw_sample['supernodes'].items():
node_idx = int(node_idx)
annotation = annotation_data['annotation']
if is_train and ignore_type_annotation(annotation):
continue
target_node_idxs.append(node_idx)
target_class.append(annotation)
target_class_id.append(
TypeClassificationModel._get_idx_for_type(
annotation, metadata, hyperparameters))
result_holder['target_node_idxs'] = np.array(target_node_idxs,
dtype=np.uint16)
result_holder['target_type'] = target_class
result_holder['variable_target_class'] = np.array(target_class_id,
dtype=np.uint16)
return len(target_node_idxs) > 0
def _load_data_from_sample(hyperparameters: Dict[str, Any],
metadata: Dict[str, Any],
raw_sample: Dict[str, Any],
result_holder: Dict[str, Any],
is_train: bool = True) -> bool:
keep_sample = super(Sequence2HybridMetric,
Sequence2HybridMetric)._load_data_from_sample(
hyperparameters, metadata, raw_sample,
result_holder, is_train)
if not keep_sample:
return False
token_node_idxs = set(raw_sample['token-sequence'])
node_idx_to_supernode_idx = {} # type: Dict[int, int]
for from_idx, to_idxs in raw_sample['edges']['OCCURRENCE_OF'].items():
from_idx = int(from_idx)
if from_idx not in token_node_idxs:
# Some supernodes do not have an associated token. Such nodes are attributes
if str(from_idx) in raw_sample['edges']['CHILD']:
right_token_idx = max(
raw_sample['edges']['CHILD'][str(from_idx)])
assert right_token_idx in token_node_idxs
from_idx = right_token_idx
else:
continue
for to_idx in to_idxs:
node_idx_to_supernode_idx[from_idx] = to_idx
supernodes_with_related_nodes = set(node_idx_to_supernode_idx.values())
variable_types = [] # type: List[str]
variable_type_idxs = [] # type: List[int]
ignored_supernodes = set()
supernode_idxs_to_annotated_variable_idx = {} # type: Dict[int, int]
for node_idx, supernode_data in raw_sample['supernodes'].items():
node_idx = int(node_idx)
annotation = supernode_data['annotation']
if ignore_type_annotation(annotation) and is_train:
ignored_supernodes.add(node_idx)
continue
if node_idx not in supernodes_with_related_nodes:
ignored_supernodes.add(node_idx)
continue
variable_idx = len(supernode_idxs_to_annotated_variable_idx)
variable_types.append(annotation)
variable_type_idxs.append(
TypeClassificationModel._get_idx_for_type(
annotation, metadata, hyperparameters))
supernode_idxs_to_annotated_variable_idx[node_idx] = variable_idx
if len(variable_types) == 0:
return False
token_idx, variable_idx = [], []
def create_token_sequence():
for i, node_idx in enumerate(raw_sample['token-sequence']):
supernode_idx = node_idx_to_supernode_idx.get(node_idx)
if supernode_idx is not None:
annotated_variable_idxs = supernode_idxs_to_annotated_variable_idx.get(
supernode_idx)
if annotated_variable_idxs is not None:
token_idx.append(i)
variable_idx.append(annotated_variable_idxs)
yield raw_sample['nodes'][node_idx]
token_sequence = list(create_token_sequence())
if len(token_sequence) > hyperparameters['max_seq_len']:
return False
# Did we see at least one token per variable?
assert len(np.unique(variable_idx)) == len(variable_types)
TokenEmbedder.load_data_from_sample('token', metadata, token_sequence,
result_holder, hyperparameters,
is_train)
result_holder['sequence_length'] = len(token_sequence)
result_holder['variable_token_idxs'] = np.array(token_idx,
dtype=np.uint32)
result_holder['variable_idxs'] = np.array(variable_idx,
dtype=np.uint32)
result_holder['target_type'] = variable_types
result_holder['variable_target_class'] = np.array(variable_type_idxs,
dtype=np.uint32)
result_holder['ignored_supernodes'] = ignored_supernodes
return keep_sample