Beispiel #1
0
def create_fixtures():
    loader = Loader('opennews',
                    project_label='Open News',
                    project_settings={},
                    source_url=DEFAULT_SOURCE_URL)

    import_schema(loader.project, StringIO(SCHEMATA))

    reader = unicodecsv.reader(StringIO(DATA))
    reader.next()

    for record in reader:
        fellow = loader.make_entity(['fellow'])
        fellow.set('name', record[0])
        fellow.set('twitter_handle', record[1])
        fellow.save()

        news_org = loader.make_entity(['news_organization'])
        news_org.set('name', record[4])
        news_org.set('url', record[5])
        news_org.save()

        fellowship = loader.make_relation('fellowship', fellow, news_org)
        fellowship.set('start_date', record[2])
        fellowship.set('end_date', record[3])
        fellowship.save()

    loader.persist()
Beispiel #2
0
def create_fixtures():
    loader = Loader('opennews', project_label='Open News',
                    project_settings={},
                    source_url=DEFAULT_SOURCE_URL)
    
    import_schema(loader.project, StringIO(SCHEMATA))

    reader = unicodecsv.reader(StringIO(DATA))
    reader.next()
     
    for record in reader:
        fellow = loader.make_entity(['fellow'])
        fellow.set('name', record[0])
        fellow.set('twitter_handle', record[1])
        fellow.save()
     
        news_org = loader.make_entity(['news_organization'])
        news_org.set('name', record[4])
        news_org.set('url', record[5])
        news_org.save()
     
        fellowship = loader.make_relation('fellowship', fellow, news_org)
        fellowship.set('start_date', record[2])
        fellowship.set('end_date', record[3])
        fellowship.save()

    loader.persist()
Beispiel #3
0
from grano.logic import Loader
import unicodecsv
 
# This source URL will be applied to all properties without their own lineage:
DEFAULT_SOURCE_URL = 'http://www.opennews.org/'
 
# Any settings (free-form dict):
PROJECT_SETTINGS = {}
 
loader = Loader('opennews', project_label='opennews',
    project_settings=PROJECT_SETTINGS, source_url=DEFAULT_SOURCE_URL)
 
reader = unicodecsv.reader(open('fellows.csv'))
reader.next()
 
for record in reader:
    fellow = loader.make_entity(['fellow'])
    fellow.set('name', record[0])
    fellow.set('twitter_handle', record[1])
    fellow.save()
 
    news_org = loader.make_entity(['news_organization'])
    news_org.set('name', record[4])
    news_org.set('url', record[5])
    news_org.save()
 
    fellowship = loader.make_relation('fellowship', fellow, news_org)
    fellowship.set('start_date', record[2])
    fellowship.set('end_date', record[3])
    fellowship.save()
Beispiel #4
0
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["id", "name", "sort_name", "email", "twitter", "facebook", "group", "group_id", "area_id", "area", "chamber", "term", "start_date", "end_date", "image", "gender"]


if __name__ == "__main__":

    loader = Loader('senegal', project_label='Senegal')
    person = loader.make_entity("politician")


    with open('senegal/senegal-politicians.csv', 'r') as csvfile:
        records_reader = csv.reader(csvfile, delimiter=",")
        for row in records_reader:
            for x in range(0, len(rows)):
                #print "%s :: %s" % (rows[x], row[x])
                if row[x]:
                    val = normalize(str(row[x]))
                    person.set( rows[x], val )
            person.save()
        loader.persist()
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["address", "source", "name", "creation_date"]

if __name__ == "__main__":

    loader = Loader('senegal', project_label='Senegal Companies')
    company = loader.make_entity("Company")

    with open('senegal/senegal-companies.csv', 'r') as csvfile:
        records_reader = csv.reader(csvfile, delimiter=",")
        for row in records_reader:
            for x in range(0, len(rows)):
                #print "%s :: %s" % (rows[x], row[x])
                if row[x]:
                    val = normalize(str(row[x]))
                    company.set(rows[x], val)
            company.save()
        loader.persist()
Beispiel #6
0
from grano.logic import Loader
import unicodecsv

# This source URL will be applied to all properties without their own lineage:
DEFAULT_SOURCE_URL = 'http://www.opennews.org/'

# Any settings (free-form dict):
PROJECT_SETTINGS = {}

loader = Loader('opennews2',
                project_label='opennews',
                project_settings=PROJECT_SETTINGS,
                source_url=DEFAULT_SOURCE_URL)

reader = unicodecsv.reader(open('fellows.csv'))
reader.next()

for record in reader:
    fellow = loader.make_entity(['fellow'])
    fellow.set('name', record[0])
    fellow.set('twitter_handle', record[1])
    fellow.save()

    news_org = loader.make_entity(['news_organization'])
    news_org.set('name', record[4])
    news_org.set('url', record[5])
    news_org.save()

    fellowship = loader.make_relation('fellowship', fellow, news_org)
    fellowship.set('start_date', record[2])
    fellowship.set('end_date', record[3])
def make_loader(source_url=None):
    return Loader('openinterests',
                  source_url=source_url,
                  project_label='OpenInterests.eu')
# coding: utf-8
from normality import normalize
import csv
import normality
from grano.logic import Loader
rows = ["address", "source", "name", "creation_date"]


if __name__ == "__main__":

    loader = Loader('senegal', project_label='Senegal Companies')
    company = loader.make_entity("Company")


    with open('senegal/senegal-companies.csv', 'r') as csvfile:
        records_reader = csv.reader(csvfile, delimiter=",")
        for row in records_reader:
            for x in range(0, len(rows)):
                #print "%s :: %s" % (rows[x], row[x])
                if row[x]:
                    val = normalize(str(row[x]))
                    company.set( rows[x], val )
            company.save()
        loader.persist()
Beispiel #9
0
def create_loader(project_slug, source_url=None, project_label=None):
    return Loader(project_slug,
                  project_label=project_label or project_slug,
                  project_settings={},
                  source_url=source_url or DEFAULT_SOURCE_URL)
Beispiel #10
0
    else:
        return node['type']


def edge_type(source, target):
    ends = (node_type(source), node_type(target))
    if ends == ('person', 'university'):
        return 'affiliation'
    elif ends == ('person', 'activity'):
        return 'participation'
    else:
        raise


loader = Loader('dhs_stanford',
                project_label='opennews',
                project_settings=PROJECT_SETTINGS,
                source_url=DEFAULT_SOURCE_URL)

graph = networkx.gexf.read_gexf('dh11.gexf')

entities = {}

for node_id in graph.nodes():
    node = graph.node[node_id]

    node_schemata = ['category']
    if node_type(node) == 'activity':
        node_schemata.append('activity')
        entity = loader.make_entity(node_schemata)
        entity.set('type', node['type'])
    elif node_type(node) == 'university':
Beispiel #11
0
def node_type(node):
    if node['type'] in ('Long Paper', 'Panel', 'Poster / Demo', 'Pre-Conference Tutorial', 'Pre-Conference Workshop', 'Short Paper'):
        return 'activity'
    else:
        return node['type']

def edge_type(source, target):
    ends = (node_type(source), node_type(target))
    if ends == ('person', 'university'):
        return 'affiliation'
    elif ends == ('person', 'activity'):
        return 'participation'
    else:
        raise

loader = Loader('dhs_stanford', project_label='opennews',
    project_settings=PROJECT_SETTINGS, source_url=DEFAULT_SOURCE_URL)

graph = networkx.gexf.read_gexf('dh11.gexf')

entities = {}

for node_id in graph.nodes():
    node = graph.node[node_id]

    node_schemata = ['category']
    if node_type(node) == 'activity':
        node_schemata.append('activity')
        entity = loader.make_entity(node_schemata)
        entity.set('type', node['type'])
    elif node_type(node) == 'university':
        node_schemata.append('university')