def test_join_instead_of_expand(join_type):
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset1 = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset',
entities_col_name='tweet')\
.expand(src_col_name='tweet', predicate_list=[('sioc:has_creater', 'tweep', False)])
dataset2 = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset',
entities_col_name='tweet')\
.expand(src_col_name='tweet', predicate_list=[('sioc:content', 'text', False)])
dataset2.join(dataset1, 'tweet', 'tweet', 'tweet', join_type)
sparql_query = dataset2.to_sparql()
print("SPARQL query with {} =\n{}\n".format(join_type, sparql_query))
def _expandable_expandable_join(join_type, optional1, optional2):
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph(
'twitter',
'https://twitter.com',
prefixes={
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset1',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[
('sioc:has_creater', 'tweep', False),
('sioc:content', 'text', optional1)
]).select_cols(['tweep'])
dataset2 = graph.entities(class_name='sioct:tweeter',
new_dataset_name='dataset2',
entities_col_name='tweeter')
dataset2 = dataset2.expand(src_col_name='tweeter',
predicate_list=[('sioc:has_name', 'name',
optional2)])
dataset.join(dataset2, 'tweep', 'tweeter', 'tweep', join_type)
return dataset.to_sparql()
def test_simple_query():
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter', 'https://twitter.com',
prefixes={
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioc:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep', False),
RDFPredicate('sioc:content', 'text', False)
])
dataset = dataset.group_by(['tweep']).count(src_col_name='tweet', new_col_name='tweet_count', unique=True)
sparql_query = dataset.to_sparql()
print("sparql_query that returns each user and his unique tweet count =\n{}\n".format(sparql_query))
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioc:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep', False),
RDFPredicate('sioc:content', 'text', False)
])
dataset = dataset.group_by(['tweep']).count('tweet')
sparql_query = dataset.to_sparql()
print("sparql_query that returns the number of tweets per user without unique =\n{}\n".format(sparql_query))
dataset = graph.entities(class_name='sioc:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep', False),
RDFPredicate('sioc:content', 'text', False)
])
dataset = dataset.count(unique=True)
sparql_query = dataset.to_sparql()
print("sparql_query that returns the number of tweets =\n{}\n".format(sparql_query))
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioc:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep', False)
])
dataset = dataset.group_by(['tweep']).count(src_col_name='tweet', new_col_name='tweet_count', unique=True)
dataset = dataset.expand(src_col_name='tweep', predicate_list=[RDFPredicate('sioc:content', 'text', False)])
sparql_query = dataset.to_sparql()
print("sparql_query that returns the tweep, tweet_count, text of each tweet =\n{}\n".format(sparql_query))
def important_vldb_authors():
"""
Returns the SPARQL query that finds all authors that have more than 20 vldb papers using dblp data.
"""
graph = KnowledgeGraph(graph_name='dblp',
graph_uri='http://dblp.l3s.de',
prefixes={
"xsd":
"http://www.w3.org/2001/XMLSchema#",
"swrc":
"http://swrc.ontoware.org/ontology#",
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"dc":
"http://purl.org/dc/elements/1.1/",
"dcterm":
"http://purl.org/dc/terms/",
"dblprc":
"http://dblp.l3s.de/d2r/resource/conferences/"
})
dataset = graph.entities(class_name='swrc:InProceedings',
new_dataset_name='papers',
entities_col_name='paper')
dataset = dataset.expand(src_col_name='paper', predicate_list=[
('dc:title', 'title'),
('dc:creator', 'author'),
('swrc:series', 'conference')])\
.filter(conditions_dict={'conference': ['= <https://dblp.l3s.de/d2r/resource/conferences/vldb>']})
grouped_dataset = dataset.group_by(['author'])\
.count('paper', 'papers_count')\
.filter(conditions_dict={'papers_count': ['>= {}'.format(20)]})
grouped_dataset = grouped_dataset.select_cols(['author', 'papers_count'])
print("SPARQL Query = \n{}".format(grouped_dataset.to_sparql()))
def test_filter_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
('sioc:has_creater', 'tweep', False),
('sioc:content', 'text', True)])\
.filter({'text': [' >= \"aa\"']})\
.select_cols(['tweet', 'text'])
# TODO: make sure the order of filter when called before a join or optional is done before the join or the optional
# and when called after the join or optional are done after it
sparql_query = dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_filter_after_group_by():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter', 'https://twitter.com',
prefixes={
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
# expand each tweet by the following features: text and tweep
ds = dataset.expand(src_col_name='tweet', predicate_list=[
('sioc:has_creater', 'tweep'),
('sioc:content', 'text')
])
# return all the tweets of users whose tweep tweeted 250-300 twweets
gds = ds.group_by(groupby_cols_list=['tweep'])\
.count('tweet', 'tweets_count')\
.filter(conditions_dict={'tweets_count': ['> {}'.format(250), '< {}'.format(300)]})
# expand these tweets by the following features: date, media, hashtags, users mentioned
# TODO: Bug. implement filter fully
gds = gds.filter({'tweep': ' >= aa'})
gds.print_query_structure()
sparql_query = gds.to_sparql()
end_transformation = time.time()
print('Transformed in {} sec'.format(end_transformation-start))
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_groupby_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(
class_name='sioct:microblogPost',
new_dataset_name='tweets',
# class_col_name='tweet_class',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[('sioc:has_creater', 'tweep',
True),
('sioc:content', 'text', False)])
dataset = dataset.group_by(['tweep'])
sparql_query = dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_groupby_aggregation_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioc:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[('sioc:has_creater', 'tweep',
False),
('sioc:content', 'text', False)])
grouped_dataset = dataset.group_by(['tweep'])\
.count('tweet', 'tweets_count')\
.select_cols(['tweep'])
# TODO: when select after groupby and aggregation, remove the non-selected columns from the select clause
# including aggregation columns
sparql_query = grouped_dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_sort_limit_offset_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[('sioc:has_creater', 'tweep',
True),
('sioc:content', 'text', False)])
dataset.sort({'tweep': 'ASC'}).limit(10).offset(5)
# TODO: do we care about limit after or before an offset? Do we allow one limit in each query?
sparql_query = dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_grouped_expandable_join(join_type):
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com/',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset1',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep',
False),
RDFPredicate('sioc:content', 'text', False)
])
dataset2 = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset2 = dataset2.expand(
src_col_name='tweet',
predicate_list=[RDFPredicate('sioc:has_creater', 'tweeter')]).group_by(
['tweeter'
]).count('tweet', 'tweets_count').filter(conditions_dict={
'tweets_count': ['>= {}'.format(200), '<= {}'.format(300)]
})
dataset2 = dataset2.expand(
src_col_name='tweeter',
predicate_list=[RDFPredicate('rdf:type', 'sioc:UserAccount')])
dataset2.join(dataset, 'tweeter', 'tweep', 'user', join_type)
dataset2.select_cols(['user'])
sparql_query = dataset2.to_sparql()
print("SPARQL query with {} =\n{}\n".format(join_type, sparql_query))
def test_users_tweets_count():
"""
In twitter dataset, retrieve all users having tweets count >= count_threshold
:return:
"""
start = time.time()
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
ds = dataset.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep'),
RDFPredicate('sioc:content', 'text'),
RDFPredicate('dcterms:created', 'date'),
RDFPredicate('to:hasmedia', 'multimedia'),
RDFPredicate('to:hashashtag', 'hashtag'),
RDFPredicate('sioc:mentions', 'users_mentioned')
])
ds = ds.expand(src_col_name='tweep',
predicate_list=[RDFPredicate('sioc:name', 'tweep_name')])
gds = ds.group_by(groupby_cols_list=['tweep'])
gds = gds.count('tweet', 'tweets_count')
gds = gds.filter(conditions_dict={
'tweets_count': ['> {}'.format(250), '< {}'.format(300)]
})
ds = ds.sort({'tweep': 'ASC'}).limit(10).offset(5)
ds = ds.select_cols([
'tweet', 'tweep', 'tweep_name', 'text', 'date', 'multimedia',
'hashtag', 'users_mentioned'
])
sparql = ds.to_sparql()
end_transformation = time.time()
print('Transformed in {} sec'.format(end_transformation - start))
print(sparql)
def test_grouped_grouped_join(join_type):
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com/',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset1',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet', predicate_list=[
('sioc:has_creater', 'tweep', False),
('sioc:content', 'text', False)])\
.group_by(['tweep']).count('tweet', 'tweets_count')\
.filter({'tweets_count': ['>= {}'.format(1000)]})
graph2 = KnowledgeGraph('twitter',
'https://twitter.com/',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc2": "http://rdfs.org/sioc2/ns#",
"sioct2": "http://rdfs.org/sioc2/types#",
})
dataset2 = graph2.entities(class_name='sioct2:twitterPost',
new_dataset_name='tweets',
entities_col_name='tweet')
dataset2 = dataset2.expand(src_col_name='tweet', predicate_list=[
('sioc2:has_creater', 'tweeter')
]).group_by(['tweeter']).count('tweet', 'tweets_count2', unique=False)\
.filter(conditions_dict={'tweets_count2': ['>= {}'.format(200), '<= {}'.format(300)]})
dataset.join(dataset2, 'tweep', 'tweeter', 'user', join_type)
dataset.select_cols(['user'])
sparql_query = dataset.to_sparql()
print("SPARQL query with {} =\n{}\n".format(join_type, sparql_query))
def important_topics():
"""
Returns the SPARQL query to identify the hot areas of research in a field of databases.
First, we identify a list of the top conferences of the computer science field of interest.
We then identify the authors who have published more than 20 papers in these conferences since the year 2000.
Next, we find the titles of all papers published by these authors in the specified conferences since 2005.
"""
graph = KnowledgeGraph(graph_name='dblp',
graph_uri='http://dblp.l3s.de',
prefixes={
"xsd":
"http://www.w3.org/2001/XMLSchema#",
"swrc":
"http://swrc.ontoware.org/ontology#",
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"dc":
"http://purl.org/dc/elements/1.1/",
"dcterm":
"http://purl.org/dc/terms/",
"dblprc":
"http://dblp.l3s.de/d2r/resource/conferences/"
})
endpoint = 'http://10.161.202.101:8890/sparql/'
port = 8890
output_format = HttpClientDataFormat.PANDAS_DF
max_rows = 1000000
timeout = 12000
client = HttpClient(endpoint_url=endpoint,
port=port,
return_format=output_format,
timeout=timeout,
max_rows=max_rows)
dataset = graph.entities('swrc:InProceedings', entities_col_name='paper')\
.expand(src_col_name='paper', predicate_list=[('dc:creator', 'author'), ('dcterm:issued', 'date'),
('swrc:series', 'conference'), ('dc:title', 'title')])
dataset = dataset.cache()
authors = dataset.filter({'date':['>= 2000'], 'conference': ['IN (dblprc:vldb, dblprc:sigmod)']})\
.group_by(['author'])\
.count('paper', 'papers_count')\
.filter({'papers_count':['>= 20']})
titles = dataset.join(authors, 'author').filter({
'date': ['>= 2005']
}).select_cols(['title'])
print("SPARQL Query = \n{}".format(titles.to_sparql()))
df = titles.execute(client, return_format=output_format)
print(df)
def test_expandable_expandable_join_w_selectcols():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset1',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep',
False),
RDFPredicate('sioc:content', 'text', False)
]).select_cols(['tweep', 'text'])
dataset2 = graph.entities(class_name='sioct:tweeter',
new_dataset_name='dataset2',
entities_col_name='tweep')
dataset2 = dataset2.expand(
src_col_name='tweep',
predicate_list=[RDFPredicate('sioc:has_name', 'name',
False)]).select_cols(['tweep', 'name'])
dataset.join(dataset2, 'tweep', 'tweep', 'tweep', JoinType.InnerJoin)
sparql_query = dataset.to_sparql()
print("SPARQL query =\n{}\n".format(sparql_query))
def test_join_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='dataset1',
entities_col_name='tweet')
dataset = dataset.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep',
False),
RDFPredicate('sioc:content', 'text', False)
])
dataset2 = graph.entities(class_name='sioc:UserAccount',
new_dataset_name='dataset2',
entities_col_name='tweep')
dataset2 = dataset2.expand(
src_col_name='tweep',
predicate_list=[RDFPredicate('sioc:has_name', 'name', False)])
# TODO: put the whole first dataset in one optional block. now, its in multiple optional blocks
dataset.join(dataset2, 'tweep', 'tweep', 'tweep', JoinType.RightOuterJoin)
sparql_query = dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
def test_twitter_query():
# TODO: remove endpoint URI
endpoint = 'http://10.161.202.101:8890/sparql/'
port = 8890
output_format = HttpClientDataFormat.PANDAS_DF
max_rows = 1000000
timeout = 12000
default_graph_url = 'http://twitter.com'
client = HttpClient(endpoint_url=endpoint,
port=port,
return_format=output_format,
timeout=timeout,
default_graph_uri=default_graph_url,
max_rows=max_rows
)
graph = KnowledgeGraph('twitter',
'http://twitter.com/',
prefixes={
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
dataset = graph.entities(class_name='sioct:microblogPost',
entities_col_name='tweet')
ds = dataset.expand(src_col_name='tweet', predicate_list=[RDFPredicate('sioc:has_creater', 'tweep')])\
.group_by(['tweep'])\
.count('tweet', 'tweets_count')\
.filter({'tweets_count': ['>= {}'.format(200), '<= {}'.format(300)]})
ds = ds.expand('tweep', [RDFPredicate('sioc:has_creater', 'tweet', directionality=PredicateDirection.INCOMING)]).\
expand('tweet', [
RDFPredicate('sioc:content', 'text', optional=False),
RDFPredicate('dcterms:created', 'date', optional=True),
RDFPredicate('to:hasmedia', 'multimedia', optional=True),
RDFPredicate('to:hashashtag', 'hashtag', optional=True),
RDFPredicate('sioc:mentions', 'users_mentioned', optional=True)
])
ds = ds.select_cols(['tweet', 'tweep', 'text', 'date', 'multimedia', 'hashtag', 'users_mentioned', 'tweets_count'])
print("Sparql Query = \n{}".format(ds.to_sparql()))
def explore_dblp():
graph = KnowledgeGraph(graph_name='dblp',
graph_uri='http://dblp.l3s.de',
prefixes={
"xsd":
"http://www.w3.org/2001/XMLSchema#",
"swrc":
"http://swrc.ontoware.org/ontology#",
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"dc":
"http://purl.org/dc/elements/1.1/",
"dcterm":
"http://purl.org/dc/terms/",
"dblprc":
"http://dblp.l3s.de/d2r/resource/conferences/"
})
endpoint = 'http://10.161.202.101:8890/sparql/'
port = 8890
output_format = HttpClientDataFormat.PANDAS_DF
max_rows = 1000000
timeout = 12000
client = HttpClient(endpoint_url=endpoint,
port=port,
return_format=output_format,
timeout=timeout,
max_rows=max_rows)
classes = graph.classes_and_freq().sort({'frequency': 'DESC'})
#class_with_max_freq = graph.classes_and_freq().max('frequency').to_sparql()
attributes_of_papers = graph.features('swrc:InProceedings')
attributes_of_papers_with_freq = graph.features_and_freq(
'swrc:InProceedings')
papers = graph.entities('swrc:InProceedings')
#papers_with_features = graph.entities_and_features('swrc:InProceedings').to_sparql()
num_papers = graph.num_entities('swrc:InProceedings')
print("{}".format(classes.to_sparql()))
df = classes.execute(client, return_format=output_format)
#print("{}".format(attributes_of_papers.to_sparql()))
#df = attributes_of_papers.execute(client, return_format=output_format)
print(df)
def test_convenience_functions():
graph = KnowledgeGraph(graph_name='dbpedia')
entities = graph.entities('dbpo:BasketballPlayer',
entities_col_name='player')
print(entities.to_sparql())
features = graph.features('dbpo:BasketballPlayer',
features_col_name='feature_uri')
print(features.to_sparql())
entities_feats = graph.entities_and_features(
'dbpo:BasketballPlayer', [('dbpp:nationality', 'nationality'),
('dbpp:birthPlace', 'place'),
('dbpp:birthDate', 'birthDate'),
('dbpp:team', 'team')])
print(entities_feats.to_sparql())
classes_freq = graph.classes_and_freq()
print(classes_freq.to_sparql())
feats_freq = graph.features_and_freq('dbpo:BasketballPlayer')
print(feats_freq.to_sparql())
n_entities = graph.num_entities('dbpo:BasketballPlayer')
print(n_entities.to_sparql())
def test_simple_query():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com/',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:MicroblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
sparql_query = dataset.to_sparql()
print("sparql_query to return tweets =\n{}\n".format(sparql_query))
endpoint = 'http://10.161.202.101:8890/sparql/'
port = 8890
output_format = HttpClientDataFormat.PANDAS_DF
max_rows = 1000000
timeout = 12000
default_graph_url = 'http://twitter.com/'
client = HttpClient(endpoint_url=endpoint,
port=port,
return_format=output_format,
timeout=timeout,
default_graph_uri=default_graph_url,
max_rows=max_rows)
#df = dataset.execute(client, return_format=output_format)
duration = start - time.time()
print("Done in {} secs".format(duration))
import pandas as pd
from rdfframes.client.http_client import HttpClientDataFormat, HttpClient
from rdfframes.knowledge_graph import KnowledgeGraph
graph = KnowledgeGraph(
graph_uri = 'http://dblp.l3s.de',
prefixes = {"xsd": "http://www.w3.org/2001/XMLSchema#",
"swrc": "http://swrc.ontoware.org/ontology#",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"dc": "http://purl.org/dc/elements/1.1/",
"dcterm": "http://purl.org/dc/terms/",
"dblprc": "http://dblp.l3s.de/d2r/resource/conferences/"})
output_format = HttpClientDataFormat.PANDAS_DF
client = HttpClient(endpoint_url=endpoint, port=port,return_format=output_format)
# RDFFrames code for creating the dataframe
papers = graph.entities('swrc:InProceedings', paper)
papers = papers.expand('paper',[('dc:creator', 'author'),('dcterm:issued', 'date'), ('swrc:series', 'conference'),
('dc:title', 'title')]).cache()
authors = papers.filter({'date': ['>=2005'],'conference': ['In(dblp:vldb, dblp:sigmod)']}).group_by(['author'])
. count('paper', 'n_papers').filter({'n_papers': '>=20', 'date': ['>=2005']})
titles = papers.join(authors, 'author', InnerJoin).select_cols(['title'])
df = titles.execute(client, return_format=output_format)
# Preprocessing and cleaning
from nltk.corpus import stopwords
df['clean_title'] = df['title'].str.replace("[^a-zA-Z#]", " ")
df['clean_title'] = df['clean_title'].apply(lambda x: x.lower())
df['clean_title'] = df['clean_title'].apply(lambda x: ' '.join([w for w in str(x).split() if len(w)>3]))
stop_words = stopwords.words('english')
tokenized_doc = df['clean_title'].apply(lambda x: x.split())
df['clean_title'] = tokenized_doc.apply(lambda x:[item for item in x if item not in stop_words])
graph = KnowledgeGraph(graph_uri='http://dbpedia.org',
prefixes= {'dcterms': 'http://purl.org/dc/terms/',
'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',
'dbpprop': 'http://dbpedia.org/property/',
'dbpr': 'http://dbpedia.org/resource/'})
endpoint = 'http://10.161.202.101:8890/sparql/'
output_format = HttpClientDataFormat.PANDAS_DF
timeout = 12000
client = HttpClient(endpoint_url=endpoint, return_format=output_format)
# RDFFrames code for creating the dataframe
songs = graph.entities('dbpo:Song', entities_col_name='song').expand('song', [('dbpp:album', 'album') ,('dbpp:artist', 'artist'),\
('dbpp:title','title'),('dbpp:lyrics', 'lyrics'), ('dbpp:writer', 'writer'), \
('dbpp:studio', 'studio'),('dbpp:genre', 'genre')])\
.expand('album', [('dbpp:title', 'Album_title') ,('dbpp:artist', 'ALbum_artist')])\
.filter({'artist': ['langMatches(lang(?artist), "en")']})
sparql_query = songs.to_sparql()
print(sparql_query)
# execution and return a dataframe of songs where artist has english name.
df = songs.execute(client, return_format=output_format)
# Preprocessing and preparation
regex = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
def test_expand_after_group_by():
start = time.time()
# create a knowledge graph to store the graph uri and prefixes
graph = KnowledgeGraph('twitter',
'https://twitter.com',
prefixes={
"rdf":
"http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/"
})
# return all the instances of the tweet class
dataset = graph.entities(class_name='sioct:microblogPost',
new_dataset_name='tweets',
entities_col_name='tweet')
sparql_query = dataset.to_sparql()
print("sparql_query 1 =\n{}\n".format(sparql_query))
# expand each tweet by the following features: text and tweep
ds = dataset.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('sioc:has_creater', 'tweep'),
RDFPredicate('sioc:content', 'text')
])
sparql_query = ds.to_sparql()
print("sparql_query 2 =\n{}\n".format(sparql_query))
# return all the tweets of users whose tweep tweeted 250-300 twweets
gds = ds.group_by(groupby_cols_list=['tweep'])\
.count('tweet', 'tweets_count')\
.filter({'tweets_count': ['> {}'.format(250), '< {}'.format(300)]})
sparql_query = gds.to_sparql()
print("sparql_query 3 =\n{}\n".format(sparql_query))
# expand these tweets by the following features: date, media, hashtags, users mentioned
gds = gds.expand(src_col_name='tweep',
predicate_list=[
RDFPredicate(
'sioc:has_creater',
'tweet',
directionality=PredicateDirection.INCOMING)
])
sparql_query = gds.to_sparql()
print("sparql_query 3.1 =\n{}\n".format(sparql_query))
gds = gds.expand(src_col_name='tweet',
predicate_list=[
RDFPredicate('dcterms:created', 'date'),
RDFPredicate('sioc:content', 'text'),
RDFPredicate('to:hasmedia', 'multimedia'),
RDFPredicate('to:hashashtag', 'hashtag'),
RDFPredicate('sioc:mentions', 'users_mentioned')
])
sparql_query = gds.to_sparql()
print("sparql_query 4 =\n{}\n\n\n\n".format(sparql_query))
# select all the tweets and their features
gds = gds.select_cols([
'tweet', 'tweep', 'text', 'date', 'multimedia', 'hashtag',
'users_mentioned'
])
# ds.print_query_structure()
gds.print_query_structure()
sparql_query = gds.to_sparql()
end_transformation = time.time()
print('Transformed in {} sec'.format(end_transformation - start))
print("sparql_query 5 =\n{}\n".format(sparql_query))