def dump_all(sources):
counter = 0
q = ''
for source in sources:
s = get_graph_spec(source)
counter += 1
q += s
if counter % 1000 == 0:
print(counter)
query = prefixes + q
q = ''
try:
helpers.send_query(endpoint_url, query)
except:
with open('data/logfile', 'w') as f:
f.write(query)
return "yikes"
try:
query = prefixes + q
helpers.send_query(endpoint_url, query)
except:
with open('data/logfile', 'w') as f:
f.write(query)
return "yikes"
print("DONE")
def write_metasources(self):
query = prefixes + """
INSERT DATA {
GRAPH <http://worldnews/metasources> {
wni:mediacloud wdt:P1448 "MediaCloud".
wni:datastreamer wdt:P1448 "DataStreamer".
wni:original wdt:P1448 "Original".
wni:inkdrop wdt:P1448 "InkDrop".
wni:wikidata wdt:P1448 "WikiData".
wni:wikipedia wdt:P1448 "Wikipedia".
wni:abyz wdt:P1448 "ABYZNewsLinks".
wni:onlineradiobox wdt:P1448 "OnlineRadioBox".
wni:w3newspapers wdt:P1448 "W3Newspapers".
wni:newscrawls wdt:P1448 "Newscrawls".
wni:topnews wdt:P1448 "Top_News".
wni:gdelt wdt:P1448 "GDELT".
wni:newsgrabber wdt:P1448 "Newsgrabber".
wni:wikinews wdt:P1448 "Wikinews".
wni:usnpl wdt:P1448 "USNPL".
wni:dmoz wdt:P1448 "DMOZ".
wni:prensaescrita wdt:P1448 "Prensa Escrita".
wni:commoncrawl wdt:P1448 "Common Crawl".
wni:lion wdt:P1448 "LION Publishers".}
} """
helpers.send_query(self.endpoint, query)
print('successfully wrote meta sources')
def dump(self, q):
if not self.overwrite:
query = prefixes + """
INSERT DATA {
""" + q + """} """
else:
query = prefixes + q
# break for bad queries
try:
helpers.send_query(self.endpoint, query)
finally:
return query
def get_all():
query = """ SELECT ?g
{GRAPH ?g {}}
"""
q = helpers.send_query(endpoint_url, query)
res = (q['results']['bindings'])
return [item['g'] for item in res]
def write_sources(outfile):
query = """SELECT ?item ?itemLabel ?country ?countryLabel ?url
WHERE
{
{?item wdt:P31 wd:Q11032;
wdt:P856 ?url} UNION
{?item wdt:P31 wd:Q1153191;
wdt:P856 ?url} UNION
{?item wdt:P31 wd:Q1110794;
wdt:P856 ?url}.
OPTIONAL { ?item wdt:P495 ?country }.
SERVICE wikibase:label { bd:serviceParam wikibase:language
"[AUTO_LANGUAGE],en". }
}"""
results = helpers.send_query(endpoint_url, query)
res = results['results']['bindings']
with open(outfile, mode='w') as f:
w = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
w.writerow(['country', 'source url', 'title', 'language', 'type'])
for item in res:
try:
country = item['countryLabel']['value']
except KeyError:
country = 'None'
title, url = item['itemLabel']['value'], item['url']['value']
w.writerow([country, url, title, "None", "None"])