def test_vector_space_wrapper(frame=None): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ # Load a VSW from a user-supplied frame if frame: frame = load_any_embeddings(frame) wrap = VectorSpaceWrapper(frame=frame) wrap.load() ok_(all(is_term(label) for label in wrap.frame.index[1:])) ok_(wrap.frame.index.is_monotonic_increasing) # Load a VSW from a filename vector_filename = DATA + '/vectors/glove12-840B.h5' wrap = VectorSpaceWrapper(vector_filename=vector_filename) wrap.load() ok_(all(is_term(label) for label in wrap.frame.index[1:])) ok_(wrap.frame.index.is_monotonic_increasing) # Load a VSW from a frame frame = load_any_embeddings(DATA + '/vectors/glove12-840B.h5') wrap = VectorSpaceWrapper(frame=frame) wrap.load() ok_(all(is_term(label) for label in wrap.frame.index[1:])) ok_(wrap.frame.index.is_monotonic_increasing)
def ld_node(uri, label=None): """ Convert a ConceptNet URI into a dictionary suitable for Linked Data. """ if label is None: label = uri_to_label(uri) ld = {'@id': uri, 'label': label} if is_term(uri): pieces = split_uri(uri) ld['language'] = get_uri_language(uri) if len(pieces) > 3: ld['sense_label'] = '/'.join(pieces[3:]) ld['term'] = uri_prefix(uri) ld['@type'] = 'Node' elif uri.startswith('http'): domain = urlparse(uri).netloc ld['site'] = domain ld['term'] = uri # OpenCyc is down and UMBEL doesn't host their vocabulary on the # Web. This property indicates whether you can follow a link # via HTTP and retrieve more information. ld['site_available'] = domain not in {'sw.opencyc.org', 'umbel.org'} ld['@type'] = 'Node' elif uri.startswith('/r/'): ld['@type'] = 'Relation' return ld
def test_standardize_row_labels(): vec1 = TEST_FRAME.loc['island'] vec2 = TEST_FRAME.loc['Island'] vec3 = TEST_FRAME.loc['thing'] standardized_vectors = standardize_row_labels(TEST_FRAME) # Check if all labels are terms ok_(all(is_term(label) for label in standardized_vectors.index)) # Check if all terms standardized to the same concept are merged ok_(standardized_vectors.index.is_unique) ok_('/c/en/Island' not in standardized_vectors.index) ok_('/c/en/island' in standardized_vectors.index) ok_('/c/en/thing' in standardized_vectors.index) ok_(standardized_vectors.loc['/c/en/island'].equals(pd.Series([3.0, 3.0, 3.0]))) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec1)) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec2)) ok_(not standardized_vectors.loc['/c/en/thing'].equals(vec3)) # Check if numbers are substituted with '#' ok_('/c/en/##' in standardized_vectors.index)
def test_vector_space_wrapper(): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ wrap = VectorSpaceWrapper(frame=TEST_FRAME) wrap.load() ok_(all(is_term(label) for label in wrap.frame.index)) ok_(wrap.frame.index.is_monotonic_increasing) # test there are no transformations to raw terms other than adding the english tag ok_('/c/en/figure skater' in wrap.frame.index) # no underscore ok_('/c/en/Island' in wrap.frame.index) # no case folding # test index_prefix_range ok_(wrap.index_prefix_range('/c/en/figure') == (3, 6)) ok_(wrap.index_prefix_range('/c/en/skating') == (0, 0)) # test_similar_terms ok_('/c/en/figure skating' in wrap.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure skater' in wrap.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure' in wrap.similar_terms('/c/en/figure skating', limit=3).index)
def test_standardize_row_labels(): vec1 = TEST_FRAME.loc['island'] vec2 = TEST_FRAME.loc['Island'] vec3 = TEST_FRAME.loc['thing'] standardized_vectors = standardize_row_labels(TEST_FRAME) # Check if all labels are terms ok_(all(is_term(label) for label in standardized_vectors.index)) # Check if all terms standardized to the same concept are merged ok_(standardized_vectors.index.is_unique) ok_('/c/en/Island' not in standardized_vectors.index) ok_('/c/en/island' in standardized_vectors.index) ok_('/c/en/thing' in standardized_vectors.index) ok_(standardized_vectors.loc['/c/en/island'].equals(pd.Series([3.0, 3.0, 3.0]))) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec1)) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec2)) ok_(not standardized_vectors.loc['/c/en/thing'].equals(vec3)) # Check if numbers are substituted with '#' ok_('/c/en/##' in standardized_vectors.index)
def ld_node(uri, label=None): """ Convert a ConceptNet URI into a dictionary suitable for Linked Data. """ if label is None: label = uri_to_label(uri) ld = { '@id': uri, 'label': label } if is_term(uri): pieces = split_uri(uri) ld['language'] = get_uri_language(uri) if len(pieces) > 3: ld['sense_label'] = '/'.join(pieces[3:]) ld['term'] = uri_prefix(uri) elif uri.startswith('http'): domain = urlparse(uri).netloc ld['site'] = domain ld['term'] = uri return ld
def test_standardize_row_labels(simple_frame): vec1 = simple_frame.loc['island'] vec2 = simple_frame.loc['Island'] vec3 = simple_frame.loc['thing'] standardized_vectors = standardize_row_labels(simple_frame) # Check if all labels are terms assert all(is_term(label) for label in standardized_vectors.index) # Check if all terms standardized to the same concept are merged assert standardized_vectors.index.is_unique assert '/c/en/Island' not in standardized_vectors.index assert '/c/en/island' in standardized_vectors.index assert '/c/en/thing' in standardized_vectors.index assert standardized_vectors.loc['/c/en/island'].equals( pd.Series([3.0, 3.0, 3.0])) assert not standardized_vectors.loc['/c/en/island'].equals(vec1) assert not standardized_vectors.loc['/c/en/island'].equals(vec2) assert not standardized_vectors.loc['/c/en/thing'].equals(vec3) # Check if numbers are substituted with '#' assert '/c/en/##' in standardized_vectors.index
def ld_node(uri, label=None): """ Convert a ConceptNet URI into a dictionary suitable for Linked Data. """ if label is None: label = uri_to_label(uri) ld = {'@id': uri, 'label': label} if is_term(uri): pieces = split_uri(uri) ld['language'] = get_uri_language(uri) # Get a reasonably-distinct sense label for the term. # Usually it will be the part of speech, but when we have fine-grained # information from Wikipedia or WordNet, it'll include the last # component as well. if len(pieces) > 3: ld['sense_label'] = pieces[3] if len(pieces) > 4 and pieces[4] in ('wp', 'wn'): ld['sense_label'] += ', ' + pieces[-1] ld['term'] = uri_prefix(uri) ld['@type'] = 'Node' elif uri.startswith('http'): domain = urlparse(uri).netloc ld['site'] = domain ld['term'] = uri # OpenCyc is down and UMBEL doesn't host their vocabulary on the # Web. This property indicates whether you can follow a link # via HTTP and retrieve more information. ld['site_available'] = True if domain in {'sw.opencyc.org', 'umbel.org', 'wikidata.dbpedia.org'}: ld['site_available'] = False ld['path'] = urlparse(uri).path ld['@type'] = 'Node' elif uri.startswith('/r/'): ld['@type'] = 'Relation' return ld
def ld_node(uri, label=None): """ Convert a ConceptNet URI into a dictionary suitable for Linked Data. """ if label is None: label = uri_to_label(uri) ld = {'@id': uri, 'label': label} if is_term(uri): pieces = split_uri(uri) ld['language'] = get_uri_language(uri) # Get a reasonably-distinct sense label for the term. # Usually it will be the part of speech, but when we have fine-grained # information from Wikipedia or WordNet, it'll include the last # component as well. if len(pieces) > 3: ld['sense_label'] = pieces[3] if len(pieces) > 4 and pieces[4] in ('wp', 'wn'): ld['sense_label'] += ', ' + pieces[-1] ld['term'] = uri_prefix(uri) ld['@type'] = 'Node' elif uri.startswith('http'): domain = urlparse(uri).netloc ld['site'] = domain ld['term'] = uri # OpenCyc is down and UMBEL doesn't host their vocabulary on the # Web. This property indicates whether you can follow a link # via HTTP and retrieve more information. ld['site_available'] = domain not in {'sw.opencyc.org', 'umbel.org'} ld['@type'] = 'Node' elif uri.startswith('/r/'): ld['@type'] = 'Relation' return ld
def test_standardize_row_labels(frame=None): if not frame: frame = DATA + '/raw/vectors/glove12.840B.300d.txt.gz' vectors = load_any_embeddings(frame) vec1 = vectors.loc['island'] vec2 = vectors.loc['Island'] vec3 = vectors.loc['things'] standardized_vectors = standardize_row_labels(vectors) # Check if all labels are terms ok_(all(is_term(label) for label in standardized_vectors.index[1:])) # Check if all terms standardized to the same concept are merged ok_(standardized_vectors.index.is_unique) ok_('/c/en/Island' not in standardized_vectors.index) ok_('/c/en/island' in standardized_vectors.index) ok_('/c/en/thing' in standardized_vectors.index) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec1)) ok_(not standardized_vectors.loc['/c/en/island'].equals(vec2)) ok_(not standardized_vectors.loc['/c/en/things'].equals(vec3)) # Check if numbers are substituted with '#' ok_('/c/en/##' in standardized_vectors.index)