def create_fixtures():
loader = Loader('opennews',
project_label='Open News',
project_settings={},
source_url=DEFAULT_SOURCE_URL)
import_schema(loader.project, StringIO(SCHEMATA))
reader = unicodecsv.reader(StringIO(DATA))
reader.next()
for record in reader:
fellow = loader.make_entity(['fellow'])
fellow.set('name', record[0])
fellow.set('twitter_handle', record[1])
fellow.save()
news_org = loader.make_entity(['news_organization'])
news_org.set('name', record[4])
news_org.set('url', record[5])
news_org.save()
fellowship = loader.make_relation('fellowship', fellow, news_org)
fellowship.set('start_date', record[2])
fellowship.set('end_date', record[3])
fellowship.save()
loader.persist()
def create_fixtures():
loader = Loader('opennews', project_label='Open News',
project_settings={},
source_url=DEFAULT_SOURCE_URL)
import_schema(loader.project, StringIO(SCHEMATA))
reader = unicodecsv.reader(StringIO(DATA))
reader.next()
for record in reader:
fellow = loader.make_entity(['fellow'])
fellow.set('name', record[0])
fellow.set('twitter_handle', record[1])
fellow.save()
news_org = loader.make_entity(['news_organization'])
news_org.set('name', record[4])
news_org.set('url', record[5])
news_org.save()
fellowship = loader.make_relation('fellowship', fellow, news_org)
fellowship.set('start_date', record[2])
fellowship.set('end_date', record[3])
fellowship.save()
loader.persist()
from grano.logic import Loader
import unicodecsv
# This source URL will be applied to all properties without their own lineage:
DEFAULT_SOURCE_URL = 'http://www.opennews.org/'
# Any settings (free-form dict):
PROJECT_SETTINGS = {}
loader = Loader('opennews', project_label='opennews',
project_settings=PROJECT_SETTINGS, source_url=DEFAULT_SOURCE_URL)
reader = unicodecsv.reader(open('fellows.csv'))
reader.next()
for record in reader:
fellow = loader.make_entity(['fellow'])
fellow.set('name', record[0])
fellow.set('twitter_handle', record[1])
fellow.save()
news_org = loader.make_entity(['news_organization'])
news_org.set('name', record[4])
news_org.set('url', record[5])
news_org.save()
fellowship = loader.make_relation('fellowship', fellow, news_org)
fellowship.set('start_date', record[2])
fellowship.set('end_date', record[3])
fellowship.save()
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["id", "name", "sort_name", "email", "twitter", "facebook", "group", "group_id", "area_id", "area", "chamber", "term", "start_date", "end_date", "image", "gender"]
if __name__ == "__main__":
loader = Loader('senegal', project_label='Senegal')
person = loader.make_entity("politician")
with open('senegal/senegal-politicians.csv', 'r') as csvfile:
records_reader = csv.reader(csvfile, delimiter=",")
for row in records_reader:
for x in range(0, len(rows)):
#print "%s :: %s" % (rows[x], row[x])
if row[x]:
val = normalize(str(row[x]))
person.set( rows[x], val )
person.save()
loader.persist()
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["address", "source", "name", "creation_date"]
if __name__ == "__main__":
loader = Loader('senegal', project_label='Senegal Companies')
company = loader.make_entity("Company")
with open('senegal/senegal-companies.csv', 'r') as csvfile:
records_reader = csv.reader(csvfile, delimiter=",")
for row in records_reader:
for x in range(0, len(rows)):
#print "%s :: %s" % (rows[x], row[x])
if row[x]:
val = normalize(str(row[x]))
company.set(rows[x], val)
company.save()
loader.persist()
from grano.logic import Loader
import unicodecsv
# This source URL will be applied to all properties without their own lineage:
DEFAULT_SOURCE_URL = 'http://www.opennews.org/'
# Any settings (free-form dict):
PROJECT_SETTINGS = {}
loader = Loader('opennews2',
project_label='opennews',
project_settings=PROJECT_SETTINGS,
source_url=DEFAULT_SOURCE_URL)
reader = unicodecsv.reader(open('fellows.csv'))
reader.next()
for record in reader:
fellow = loader.make_entity(['fellow'])
fellow.set('name', record[0])
fellow.set('twitter_handle', record[1])
fellow.save()
news_org = loader.make_entity(['news_organization'])
news_org.set('name', record[4])
news_org.set('url', record[5])
news_org.save()
fellowship = loader.make_relation('fellowship', fellow, news_org)
fellowship.set('start_date', record[2])
fellowship.set('end_date', record[3])
def make_loader(source_url=None):
return Loader('openinterests',
source_url=source_url,
project_label='OpenInterests.eu')
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["address", "source", "name", "creation_date"]
if __name__ == "__main__":
loader = Loader('senegal', project_label='Senegal Companies')
company = loader.make_entity("Company")
with open('senegal/senegal-companies.csv', 'r') as csvfile:
records_reader = csv.reader(csvfile, delimiter=",")
for row in records_reader:
for x in range(0, len(rows)):
#print "%s :: %s" % (rows[x], row[x])
if row[x]:
val = normalize(str(row[x]))
company.set( rows[x], val )
company.save()
loader.persist()
def create_loader(project_slug, source_url=None, project_label=None):
return Loader(project_slug,
project_label=project_label or project_slug,
project_settings={},
source_url=source_url or DEFAULT_SOURCE_URL)
else:
return node['type']
def edge_type(source, target):
ends = (node_type(source), node_type(target))
if ends == ('person', 'university'):
return 'affiliation'
elif ends == ('person', 'activity'):
return 'participation'
else:
raise
loader = Loader('dhs_stanford',
project_label='opennews',
project_settings=PROJECT_SETTINGS,
source_url=DEFAULT_SOURCE_URL)
graph = networkx.gexf.read_gexf('dh11.gexf')
entities = {}
for node_id in graph.nodes():
node = graph.node[node_id]
node_schemata = ['category']
if node_type(node) == 'activity':
node_schemata.append('activity')
entity = loader.make_entity(node_schemata)
entity.set('type', node['type'])
elif node_type(node) == 'university':
def node_type(node):
if node['type'] in ('Long Paper', 'Panel', 'Poster / Demo', 'Pre-Conference Tutorial', 'Pre-Conference Workshop', 'Short Paper'):
return 'activity'
else:
return node['type']
def edge_type(source, target):
ends = (node_type(source), node_type(target))
if ends == ('person', 'university'):
return 'affiliation'
elif ends == ('person', 'activity'):
return 'participation'
else:
raise
loader = Loader('dhs_stanford', project_label='opennews',
project_settings=PROJECT_SETTINGS, source_url=DEFAULT_SOURCE_URL)
graph = networkx.gexf.read_gexf('dh11.gexf')
entities = {}
for node_id in graph.nodes():
node = graph.node[node_id]
node_schemata = ['category']
if node_type(node) == 'activity':
node_schemata.append('activity')
entity = loader.make_entity(node_schemata)
entity.set('type', node['type'])
elif node_type(node) == 'university':
node_schemata.append('university')
