Example #1
0
 def convert(self, namespace):
     '''Converts the JSON and XML attributes into a RDFLib graph.
     
     Parameters
     @namespace: A RDFLib Namespace.
     
     Returns
     @graph: The RDFLib Graph of the provenance.'''
     
     if (self.defaults['id'] == None) or (self.defaults['id'] == ""):
         return None
     
     converter = IatiElements.ProvenanceElements(self.defaults, namespace)
     
     for entry in self.json:
         
         try:
             funcname = entry.replace("-","_").replace("id","func_id").replace("version", "func_version")
         
             update = getattr(converter, funcname)
             update(self.json[entry])
             
         except AttributeError as e:
             print "Error in " + funcname + ", " + self.defaults['id'] + ": " + str(e)
     
     # Add prov model
     start_time = datetime.datetime.now()
     
     if self.defaults['type'] == 'activity':
         script = "conversion%20scripts/ActivitiesToTurtle.py"
     elif self.defaults['type'] == 'organisation':
         script = "conversion%20scripts/OrganisationsToTurtle.py"
     else:
         script = None
             
     provenance = converter.get_result()
     
     provenance = AddProvenance.addProv(namespace,
                                        provenance,
                                        self.defaults['type'],
                                        self.defaults['document_name'],
                                        start_time,
                                        self.json['download_url'],
                                        self.ids,
                                        script)
     
     return provenance      
def main():
    '''Retrieves all locations from a local file and matches the labels, country labels and
    coordinates (if available) of a location to Geonames.'''

    # Settings
    turtle_folder = "/media/Acer/School/IATI-data/mappings/Geonames/"
    locations_file = "/media/Acer/School/IATI-data/mappings/locations.help"
    Iati = Namespace("http://purl.org/collections/iati/")
    start_time = datetime.datetime.now()

    found = 0
    not_found = 0

    # Read location file
    print "Retrieving locations from file..."
    all_locations = retrieve_locations(locations_file)

    # Classify locations
    print "Classifying locations..."
    all_locations = classify_locations(all_locations)

    # Initialize graph
    locations_graph = Graph()
    locations_graph.bind('iati', "http://purl.org/collections/iati/")
    locations_graph.bind('gn', "http://sws.geonames.org/")
    locations_graph.bind('owl', "http://www.w3.org/2002/07/owl#")

    # Retrieve all general country information
    country_info = connect(
        "http://api.geonames.org/countryInfo?username=kasperbrandt")
    country_info = ET.fromstring(country_info)

    if country_info == None:
        print "Could not retrieve country information, exiting..."
        sys.exit(0)

    # Retrieve location match from Geonames
    for location in all_locations:

        if not location['classification'] == 0:
            print "Looking for " + location['link'] + "..."
            match = find_location(location, country_info)

            if (not match == None) and (not match == 0):
                locations_graph.add(
                    (URIRef(location['link']), OWL.sameAs, URIRef(match)))

                found += 1

            elif match == 0:
                # Did not find any results
                not_found += 1

            elif match == None:
                # No more credits
                print "Credits are gone.."
                locations_turtle = locations_graph.serialize(format='turtle')

                with open(turtle_folder + 'geonames-locations.ttl',
                          'w') as turtle_file:
                    turtle_file.write(locations_turtle)

                sys.exit(0)

    # Write to file
    print "Done, writing " + str(found) + " mappings to file..."
    locations_turtle = locations_graph.serialize(format='turtle')

    with open(turtle_folder + 'geonames-locations.ttl', 'w') as turtle_file:
        turtle_file.write(locations_turtle)

    print "Did not find " + str(not_found) + " mappings..."

    # Add provenance
    provenance = Graph()

    provenance = AddProvenance.addProv(
        Iati, provenance, 'Geonames', start_time, "http://sws.geonames.org/",
        ['Geonames'], "mapping%20scripts/GeonamesLocations.py")

    provenance_turtle = provenance.serialize(format='turtle')

    with open(turtle_folder + 'provenance-geonames-locations.ttl',
              'w') as turtle_file:
        turtle_file.write(provenance_turtle)

    print
    print "Done, added provenance."
Example #3
0
countries_turtle_ecb = countries_ecb.serialize(format='turtle')

with open(turtle_folder_ecb + 'ecb-countries.ttl', 'w') as turtle_file_ecb:
    turtle_file_ecb.write(countries_turtle_ecb)

countries_turtle_fao = countries_fao.serialize(format='turtle')

with open(turtle_folder_fao + 'fao-countries.ttl', 'w') as turtle_file_fao:
    turtle_file_fao.write(countries_turtle_fao)

# Add provenance OECD
provenance_oecd = Graph()

provenance_oecd = AddProvenance.addProv(Iati, provenance_oecd, 'OECD',
                                        start_time, "http://dbpedia.org/",
                                        ['OECD'],
                                        "mapping%20scripts/OecdCountries.py")

provenance_turtle_oecd = provenance_oecd.serialize(format='turtle')

with open(turtle_folder_oecd + 'provenance-oecd.ttl', 'w') as turtle_file_oecd:
    turtle_file_oecd.write(provenance_turtle_oecd)

# Add provenance BFS
provenance_bfs = Graph()

provenance_bfs = AddProvenance.addProv(Iati, provenance_bfs, 'BFS', start_time,
                                       "http://dbpedia.org/", ['BFS'],
                                       "mapping%20scripts/OecdCountries.py")

provenance_turtle_bfs = provenance_bfs.serialize(format='turtle')
                    .replace(">", "")
                )
            else:
                factbook_item = line_list[2].replace("cia:", "")

            factbook_url = "http://wifo5-04.informatik.uni-mannheim.de/factbook/data/" + factbook_item
            sources.append(factbook_url)

            turtle_data = urllib2.urlopen(factbook_url).read()

            print "Retrieved data from " + factbook_url + ", writing to file..."

            with open(factbook_folder + factbook_item + ".ttl", "w") as turtle_f:
                turtle_f.write(turtle_data)

# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(
    Iati, provenance, "Factbook", start_time, sources, ["Factbook"], "gather%20data%20scripts/FactbookData.py"
)

provenance_turtle = provenance.serialize(format="turtle")

with open(factbook_folder + "provenance-factbook.ttl", "w") as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)

print "Done!"
def main():
    '''Converts Codelist XMLs to Turtle files and stores these to local folder.'''
    
    # Settings
    xml_folder = "/home/iati/xml/codelists/"
    turtle_folder = "/home/iati/codelist/"
    Iati = Namespace("http://purl.org/collections/iati/")
        
    if not os.path.isdir(turtle_folder):
        os.makedirs(turtle_folder)
    
    document_count = 1
    
    total_elapsed_time = 0
    
    # Retrieve XML files from the XML folder
    for document in glob.glob(xml_folder + '*.xml'):
        
        doc_id = str(document.rsplit('/',1)[1])[:-4]
        doc_folder = turtle_folder + doc_id + '/'
        
        if not os.path.isdir(doc_folder):
            os.makedirs(doc_folder)
        
        provenance = Graph()
        provenance.bind('iati', Iati)
        
        xml = ET.parse(document)
        root = xml.getroot()
        
        version = AttributeHelper.attribute_key(root, 'version')
        
        try:
            # Convert each codelist in XML file to RDFLib Graph    
            converter = IatiConverter.ConvertCodelist(root)
            graph, id, last_updated = converter.convert(Iati)
        except TypeError as e:
            print "Error in " + document + ":" + str(e)
            graph = None
        
        if not graph == None:
            # Write codelist to Turtle and store in local folder
            graph_turtle = graph.serialize(format='turtle')
            
            with open(doc_folder + id.replace('/','%2F') + '.ttl', 'w') as turtle_file:
                turtle_file.write(graph_turtle)
                
            # The following outputs enable the Virutuoso Bulk loader process to put files into the right graphs.
             with open(doc_folder + id.replace('/','%2F') + '.ttl.graph','w') as graph_file:
                 graph_file.write(str(Iati) + 'graph/codelist/' + str(id))
            
            # Add provenance of last-updated, version and source document
            provenance.add((URIRef(Iati + 'graph/codelist/' + str(id)),
                            URIRef(Iati + 'last-updated'),
                            Literal(last_updated)))
            
            provenance.add((URIRef(Iati + 'graph/codelist/' + str(id)),
                            URIRef(Iati + 'version'),
                            Literal(version)))
            
            provenance.add((URIRef(Iati + 'graph/codelist/' + str(id)),
                            URIRef(Iati + 'source-document-id'),
                            Literal(str(id))))
            
            provenance.add((URIRef(Iati + 'graph/codelist/' + str(id)),
                            URIRef(Iati + 'source-document-download-url'),
                            URIRef('http://datadev.aidinfolabs.org/data/codelist/' + str(id) + '.xml')))               
        
        print "Progress: Document #" + str(document_count)
                   
        document_count += 1
        
        # Add prov model
        start_time = datetime.datetime.now()
        source_xml = str('http://datadev.aidinfolabs.org/data/codelist/' + str(id) + '.xml')
        entities = []
        entities.append(str(id))
        script = "conversion%20scripts/CodelistsToTurtle.py"
        
        provenance = AddProvenance.addProv(Iati, provenance, 'codelist', doc_id, start_time, source_xml, entities, script)
    
        # Write provenance graph to Turtle and store in local folder
        provenance_turtle = provenance.serialize(format='turtle')
        
        with open(doc_folder + 'provenance-' + str(id) + '.ttl', 'w') as turtle_file:
            turtle_file.write(provenance_turtle)

        # The following outputs enable the Virutuoso Bulk loader process to put files into the right graphs.
        with open(doc_folder + 'provenance-' + str(id) + '.ttl.graph','w') as graph_file:
            graph_file.write(str(Iati) + 'graph/provenance/')
   
    print "Done!"
def main():
    '''Retrieves all locations from a local file and matches the labels, country labels and
    coordinates (if available) of a location to Geonames.'''
    
    # Settings
    turtle_folder = "/home/iati/mappings/Geonames/"
    locations_file = "/home/iati/mappings/locations.help"
    Iati = Namespace("http://purl.org/collections/iati/")
    start_time = datetime.datetime.now()
    
    found = 0
    not_found = 0
    
    # Read location file
    print "Retrieving locations from file..."
    all_locations = retrieve_locations(locations_file)
    
    # Classify locations
    print "Classifying locations..."
    all_locations = classify_locations(all_locations)
    
    # Initialize graph
    locations_graph = Graph()
    locations_graph.bind('iati', "http://purl.org/collections/iati/")
    locations_graph.bind('gn', "http://sws.geonames.org/")
    locations_graph.bind('owl', "http://www.w3.org/2002/07/owl#")
    
    # Retrieve all general country information
    country_info = connect("http://api.geonames.org/countryInfo?username=kasperbrandt")
    country_info = ET.fromstring(country_info)
    
    if country_info == None:
        print "Could not retrieve country information, exiting..."
        sys.exit(0)
    
    # Retrieve location match from Geonames
    for location in all_locations:
        
        if not location['classification'] == 0:
            print "Looking for " + location['link'] + "..."
            match = find_location(location, country_info)
            
            if (not match == None) and (not match == 0):
                locations_graph.add((URIRef(location['link']),
                                     OWL.sameAs,
                                     URIRef(match)))
                
                found += 1
                
            elif match == 0:
                # Did not find any results
                not_found += 1
            
            elif match == None:
                # No more credits
                print "Credits are gone.."
                locations_turtle = locations_graph.serialize(format='turtle')
    
                with open(turtle_folder + 'geonames-locations.ttl', 'w') as turtle_file:
                    turtle_file.write(locations_turtle)
                    
                sys.exit(0)
        
    # Write to file
    print "Done, writing " + str(found) + " mappings to file..."    
    locations_turtle = locations_graph.serialize(format='turtle')

    with open(turtle_folder + 'geonames-locations.ttl', 'w') as turtle_file:
        turtle_file.write(locations_turtle)
    
    print "Did not find " + str(not_found) + " mappings..."
    
    # Add provenance
    provenance = Graph()
    
    provenance = AddProvenance.addProv(Iati,
                                       provenance,
                                       'Geonames',
                                       start_time,
                                       "http://sws.geonames.org/",
                                       ['Geonames'],
                                       "mapping%20scripts/GeonamesLocations.py")
    
    provenance_turtle = provenance.serialize(format='turtle')

    with open(turtle_folder + 'provenance-geonames-locations.ttl', 'w') as turtle_file:
        turtle_file.write(provenance_turtle)
    
    print    
    print "Done, added provenance."
    print "Adding to file..."
    print

    turtle = data.serialize(format='turtle')
    with open(worldbank_folder + 'worldbank-' + indicator + '.ttl',
              'w') as turtle_file:
        turtle_file.write(turtle)

print "Adding indicator data to file..."
print

turtle = indicator_data.serialize(format='turtle')
with open(worldbank_folder + 'worldbank-indicators.ttl', 'w') as turtle_file:
    turtle_file.write(turtle)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'WorldBank', start_time,
                                   sources, ['WorldBank'],
                                   "gather%20data%20scripts/WorldbankData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(worldbank_folder + 'provenance-worldbank.ttl',
          'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)

print "Done! " + str(errors) + " errors."
                        total_count) + " of " + str(total_from_file) + ")..."

                    with open(
                            geonames_folder + str(geonames_item_id) +
                            "-about.rdf", 'w') as turtle_f:
                        turtle_f.write(rdf_about_data)

                    print "Retrieved data from " + geonames_contains_url + ", writing to file (" + str(
                        total_count) + " of " + str(total_from_file) + ")..."

                    with open(
                            geonames_folder + str(geonames_item_id) +
                            "-contains.rdf", 'w') as turtle_f:
                        turtle_f.write(rdf_contains_data)

# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'Geonames', start_time,
                                   source_rdfs, ['Geonames'],
                                   "gather%20data%20scripts/GeonamesData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(geonames_folder + 'provenance-geonames.ttl',
          'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)

print "Done!"
Example #9
0
        not_found.append((name, code))
        not_found_countries += 1

# Adding mappings to file
print
print "Adding to file..."
countries_turtle = countries.serialize(format='turtle')

with open(turtle_folder + 'transparency-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(
    Iati, provenance, 'Transparency', start_time, "", ['Transparency'],
    "mapping%20scripts/TransparencyCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-transparency.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)

print
print "Added provenance..."

print
print "Total: " + str(total_countries)
print "Done, found: " + str(found_countries) + ", not found: " + str(
    not_found_countries) + "."
Example #10
0
            factbook_url = "http://wifo5-04.informatik.uni-mannheim.de/factbook/data/" + factbook_item
            sources.append(factbook_url)
            
            turtle_data = urllib2.urlopen(factbook_url).read()
            
            print "Retrieved data from " + factbook_url + ", writing to file..."
            
            with open(factbook_folder + factbook_item + ".ttl", 'w') as turtle_f:
                turtle_f.write(turtle_data)

# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Factbook',
                                   start_time,
                                   sources,
                                   ['Factbook'],
                                   "gather%20data%20scripts/FactbookData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(factbook_folder + 'provenance-factbook.ttl', 'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)
    
print "Done!"
                
                
    else:
        not_found_again.append((not_found_country[0], not_found_country[1]))

print "Adding to file..."
countries_turtle = countries.serialize(format='turtle')

with open(turtle_folder + 'geonames-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Geonames',
                                   start_time,
                                   "http://sws.geonames.org/",
                                   ['Geonames'],
                                   "mapping%20scripts/GeonamesCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-geonames-countries.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)

print
print "Total countries: " + str(total)
print "Done, found: " + str(found_count) + ", not found: " + str(not_found_count) + "."

print
print "Could not find:"
Example #12
0
with open(turtle_folder + 'schema-foaf.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_foaf)
    
with open(turtle_folder + 'schema-org.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_org)
    
with open(turtle_folder + 'schema-geo.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_geo)
    
with open(turtle_folder + 'schema-cc.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_cc) 

print "Adding provenance..."

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Schema',
                                   start_time,
                                   "",
                                   ['Schema'],
                                   "mapping%20scripts/ExternalMappings.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-schema.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)

print "Done!"
Example #13
0
countries_turtle = countries.serialize(format='turtle')

with open(turtle_folder + 'factbook-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)

countries_turtle_db = countries_db.serialize(format='turtle')

with open(turtle_folder_db + 'dbpedia-countries-via-factbook.ttl',
          'w') as turtle_file_db:
    turtle_file_db.write(countries_turtle_db)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'Factbook', start_time,
                                   "http://dbpedia.org/", ['Factbook'],
                                   "mapping%20scripts/FactbookCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-factbook-countries.ttl',
          'w') as turtle_file:
    turtle_file.write(provenance_turtle)

# Add provenance
provenance_db = Graph()

provenance_db = AddProvenance.addProv(
    Iati, provenance_db, 'DBPedia', start_time, "http://dbpedia.org/",
    ['DBPedia'], "mapping%20scripts/FactbookCountries.py")
    with open(worldbank_folder + 'worldbank-'+ indicator +'.ttl', 'w') as turtle_file:
        turtle_file.write(turtle)
    
    
    
print "Adding indicator data to file..."
print

turtle = indicator_data.serialize(format='turtle')
with open(worldbank_folder + 'worldbank-indicators.ttl', 'w') as turtle_file:
    turtle_file.write(turtle)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'WorldBank',
                                   start_time,
                                   sources,
                                   ['WorldBank'],
                                   "gather%20data%20scripts/WorldbankData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(worldbank_folder + 'provenance-worldbank.ttl', 'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)
    
print "Done! " + str(errors) + " errors."
                
    
Example #15
0
print "Adding to file..."
countries_turtle = countries.serialize(format='turtle')

with open(turtle_folder + 'worldbank-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)

countries_turtle_euro = countries_euro.serialize(format='turtle')

with open(turtle_folder_euro + 'eurostat-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'WorldBank', start_time,
                                   "", ['WorldBank'],
                                   "mapping%20scripts/WorldbankCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-worldbank.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)

# Add provenance
provenance_euro = Graph()

provenance_euro = AddProvenance.addProv(
    Iati, provenance_euro, 'EuroStat', start_time, "", ['EuroStat'],
    "mapping%20scripts/WorldbankCountries.py")

provenance_turtle_euro = provenance_euro.serialize(format='turtle')
                source_ttls.append(dbpedia_url)
                
                turtle_response = urllib2.urlopen(dbpedia_url)
                turtle_data = turtle_response.read()
                
                print "Retrieved data from " + dbpedia_url + ", writing to file..."
                
                with open(dbpedia_folder + dbpedia_item + ".ttl", 'w') as turtle_f:
                    turtle_f.write(turtle_data)

# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'DBPedia',
                                   start_time,
                                   source_ttls,
                                   ['DBPedia'],
                                   "gather%20data%20scripts/DbpediaData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(dbpedia_folder + 'provenance-dbpedia.ttl', 'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)
    
print "Done!"
                
                
with open(turtle_folder + 'factbook-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)
    
countries_turtle_db = countries_db.serialize(format='turtle')

with open(turtle_folder_db + 'dbpedia-countries-via-factbook.ttl', 'w') as turtle_file_db:
    turtle_file_db.write(countries_turtle_db)
    

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Factbook',
                                   start_time,
                                   "http://dbpedia.org/",
                                   ['Factbook'],
                                   "mapping%20scripts/FactbookCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-factbook-countries.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)
    
# Add provenance
provenance_db = Graph()

provenance_db = AddProvenance.addProv(Iati,
                                      provenance_db,
                                      'DBPedia',
# Adding mappings to file
print
print "Adding to file..."
countries_turtle = countries.serialize(format='turtle')

with open(turtle_folder + 'transparency-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)
    
# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Transparency',
                                   start_time,
                                   "",
                                   ['Transparency'],
                                   "mapping%20scripts/TransparencyCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-transparency.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)
    
print
print "Added provenance..."

print
print "Total: " + str(total_countries)
print "Done, found: " + str(found_countries) + ", not found: " + str(not_found_countries) + "."
with open(turtle_folder + 'worldbank-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)
    
countries_turtle_euro = countries_euro.serialize(format='turtle')

with open(turtle_folder_euro + 'eurostat-countries.ttl', 'w') as turtle_file:
    turtle_file.write(countries_turtle)
    
# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'WorldBank',
                                   start_time,
                                   "",
                                   ['WorldBank'],
                                   "mapping%20scripts/WorldbankCountries.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-worldbank.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)
    
# Add provenance
provenance_euro = Graph()

provenance_euro = AddProvenance.addProv(Iati,
                                        provenance_euro,
                                        'EuroStat',
Example #20
0
                else:
                    dbpedia_item = line_list[2].replace("dbpedia:", "")

                dbpedia_url = "http://dbpedia.org/data/" + dbpedia_item + ".ttl"
                source_ttls.append(dbpedia_url)

                turtle_response = urllib2.urlopen(dbpedia_url)
                turtle_data = turtle_response.read()

                print "Retrieved data from " + dbpedia_url + ", writing to file..."

                with open(dbpedia_folder + dbpedia_item + ".ttl",
                          'w') as turtle_f:
                    turtle_f.write(turtle_data)

# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'DBPedia', start_time,
                                   source_ttls, ['DBPedia'],
                                   "gather%20data%20scripts/DbpediaData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(dbpedia_folder + 'provenance-dbpedia.ttl', 'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)

print "Done!"
Example #21
0
with open(turtle_folder + 'schema-dct.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_dct)

with open(turtle_folder + 'schema-foaf.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_foaf)

with open(turtle_folder + 'schema-org.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_org)

with open(turtle_folder + 'schema-geo.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_geo)

with open(turtle_folder + 'schema-cc.ttl', 'w') as turtle_file:
    turtle_file.write(graph_turtle_cc)

print "Adding provenance..."

# Add provenance
provenance = Graph()

provenance = AddProvenance.addProv(Iati, provenance, 'Schema', start_time, "",
                                   ['Schema'],
                                   "mapping%20scripts/ExternalMappings.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(turtle_folder + 'provenance-schema.ttl', 'w') as turtle_file:
    turtle_file.write(provenance_turtle)

print "Done!"
                    rdf_contains_data = urllib2.urlopen(geonames_contains_url).read()
                    
                    print "Retrieved data from " + geonames_about_url + ", writing to file (" + str(total_count) + " of " + str(total_from_file) + ")..."
                    
                    with open(geonames_folder + str(geonames_item_id) + "-about.rdf", 'w') as turtle_f:
                        turtle_f.write(rdf_about_data)
                        
                    print "Retrieved data from " + geonames_contains_url + ", writing to file (" + str(total_count) + " of " + str(total_from_file) + ")..."
                    
                    with open(geonames_folder + str(geonames_item_id) + "-contains.rdf", 'w') as turtle_f:
                        turtle_f.write(rdf_contains_data)
                    
# Add provenance
print "Adding provenance..."

provenance = Graph()

provenance = AddProvenance.addProv(Iati,
                                   provenance,
                                   'Geonames',
                                   start_time,
                                   source_rdfs,
                                   ['Geonames'],
                                   "gather%20data%20scripts/GeonamesData.py")

provenance_turtle = provenance.serialize(format='turtle')

with open(geonames_folder + 'provenance-geonames.ttl', 'w') as turtle_file_prov:
    turtle_file_prov.write(provenance_turtle)
    
print "Done!"
Example #23
0
with open(turtle_folder_ecb + 'ecb-countries.ttl', 'w') as turtle_file_ecb:
    turtle_file_ecb.write(countries_turtle_ecb)
    
countries_turtle_fao = countries_fao.serialize(format='turtle')

with open(turtle_folder_fao + 'fao-countries.ttl', 'w') as turtle_file_fao:
    turtle_file_fao.write(countries_turtle_fao)

# Add provenance OECD
provenance_oecd = Graph()

provenance_oecd = AddProvenance.addProv(Iati,
                                      provenance_oecd,
                                      'OECD',
                                      start_time,
                                      "http://dbpedia.org/",
                                      ['OECD'],
                                      "mapping%20scripts/OecdCountries.py")

provenance_turtle_oecd = provenance_oecd.serialize(format='turtle')

with open(turtle_folder_oecd + 'provenance-oecd.ttl', 'w') as turtle_file_oecd:
    turtle_file_oecd.write(provenance_turtle_oecd)
    
# Add provenance BFS
provenance_bfs = Graph()

provenance_bfs = AddProvenance.addProv(Iati,
                                      provenance_bfs,
                                      'BFS',