def calculate_musicbrainz_works_and_recordings_by_id(id, author, output_file):
try:
# query_work = MUSICBRAINZ_API_URL + 'artist/' + id + '?inc=aliases%20works%20recordings&fmt=json'
query_work = MUSICBRAINZ_API_URL + 'work?artist=' + id + '&inc=aliases&fmt=json'
print 'query compositions:', query_work
work_response = common.process_http_query(query_work)
print 'musicbrainz composition:', work_response
musicbrainz_composition_response_json = json.loads(work_response.content)
# works_count = len(musicbrainz_composition_response_json[common.WORKS_JSON])
compositions_count = str(musicbrainz_composition_response_json[common.WORK_COUNT_JSON])
#recordings_count = len(musicbrainz_composition_response_json[common.RECORDINGS_JSON])
#compositions_count = works_count + recordings_count
print 'musicbrainz #composition:', compositions_count
values = [
id
, common.toByteStr(author)
, str(compositions_count)
]
entry = dict(zip(common.musicbrainz_compositions_count_fieldnames, values))
with open(output_file, 'ab') as csvfile:
writer = csv.DictWriter(csvfile, delimiter=';', fieldnames=common.musicbrainz_compositions_count_fieldnames, lineterminator='\n')
writer.writerow(entry)
except ValueError as ve:
print 'Could not find JSON for given Musicbrainz composition.', id, ve.message
except Exception as e:
print 'Could not find Musicbrainz composition.', id, e.message
return work_response
def retrieve_wikidata_author_id(gnd):
query = WIKIDATA_API_URL + 'string[' + str(GND_ID_PROP) + ':' + gnd + ']'
print 'query:', query
wikidata_author_id_response = common.process_http_query(query)
print 'response content:', wikidata_author_id_response.content
return wikidata_author_id_response
def retrieve_wikidata_author_id(gnd):
query = TMP_WIKIDATA_API_URL + 'string[' + str(GND_ID_PROP) + ':' + gnd + ']'
print 'query:', query
wikidata_author_id_response = common.process_http_query(query)
print 'response content:', wikidata_author_id_response.content
return wikidata_author_id_response
def retrieve_wikidata_property(property_id):
query_property = WIKIDATA_API_ACTION + 'wbgetentities&ids=P' + str(property_id) + FORMAT_JSON
print 'query property:', query_property
response_json = common.process_http_query(query_property)
print 'json data:', response_json
return response_json
def retrieve_wikidata_occupation(wikidata_occupation_id):
query_occupation = WIKIDATA_API_URL + str(wikidata_occupation_id) + LANGUAGE_EN + FORMAT_JSON
print 'query occupation:', query_occupation
occupation_response_json = common.process_http_query(query_occupation)
print 'occupation json data:', occupation_response_json
return occupation_response_json
def retrieve_wikidata_author_data(wikidata_author_id):
query_author = WIKIDATA_API_URL + str(wikidata_author_id) + LANGUAGE_EN + FORMAT_JSON
print 'query author:', query_author
author_response_json = common.process_http_query(query_author)
print 'author json data:', author_response_json
return author_response_json
def retrieve_wikidata_object_by_internet_archive_id(internet_archive_id):
query = WIKIDATA_API_URL + 'string[' + str(
INTERNET_ARCHIVE_ID_PROP) + ':' + internet_archive_id + ']'
print 'query wikidata object:', query
wikidata_object_response = common.process_http_query(query)
print 'response wikidata object content:', wikidata_object_response.content
return wikidata_object_response
def retrieve_wikidata_composition_by_freebase_id(freebase_id):
query = WIKIDATA_API_URL + 'string[' + str(
FREEBASE_ID_PROP) + ':' + freebase_id + ']'
print 'query wikidata composition:', query
wikidata_composition_response = common.process_http_query(query)
print 'response wikidata composition content:', wikidata_composition_response.content
return wikidata_composition_response
def retrieve_wikidata_author_data(wikidata_author_id):
query_author = WIKIDATA_API_URL + str(
wikidata_author_id) + LANGUAGE_EN + FORMAT_JSON
print 'query author:', query_author
author_response_json = common.process_http_query(query_author)
print 'author json data:', author_response_json
return author_response_json
def search_europeana_facets():
europeana_response = common.process_http_query(EUROPEANA_COLLECTION_URL)
print 'response content:', europeana_response.content
europeana_response_json = europeana_response.json()
# facets -> fields -> label
labels = extract_and_save_label_data(europeana_response_json)
print 'labels len', len(labels)
def retrieve_wikidata_occupation(wikidata_occupation_id):
query_occupation = WIKIDATA_API_URL + ITEMS_JSON + '[' + str(wikidata_occupation_id) + ']&' + \
PROPS_JSON + '=' + str(COMMONS_CATEGORY_PROP)
print 'query occupation:', query_occupation
occupation_response_json = common.process_http_query(query_occupation)
print 'occupation json data:', occupation_response_json
return occupation_response_json
def retrieve_wikidata_author_data(wikidata_author_id):
query_author = WIKIDATA_API_URL + ITEMS_JSON + '[' + str(wikidata_author_id) + ']&' + \
PROPS_JSON + '=' + ", ".join(str(e) for e in properties)
print 'query author:', query_author
author_response_json = common.process_http_query(query_author)
print 'author json data:', author_response_json
return author_response_json
def retrieve_wikidata_band_data(wikidata_band_id):
query_band = WIKIDATA_API_URL + ITEMS_JSON + '[' + str(wikidata_band_id) + ']&' + \
PROPS_JSON + '=*'
print 'query band:', query_band
band_response_json = common.process_http_query(query_band)
print 'band json data:', band_response_json
return band_response_json
def retrieve_wikidata_property(property_id):
query_property = WIKIDATA_API_ACTION + 'wbgetentities&ids=P' + str(
property_id) + FORMAT_JSON
print 'query property:', query_property
response_json = common.process_http_query(query_property)
print 'json data:', response_json
return response_json
def retrieve_wikidata_occupation(wikidata_occupation_id):
query_occupation = WIKIDATA_API_URL + str(
wikidata_occupation_id) + LANGUAGE_EN + FORMAT_JSON
print 'query occupation:', query_occupation
occupation_response_json = common.process_http_query(query_occupation)
print 'occupation json data:', occupation_response_json
return occupation_response_json
def retrieve_wikidata_composition_by_musicbrainz_id(musicbrainz_id):
query = WIKIDATA_API_URL + 'string[' + str(
MUSICBRAINZ_COMPOSITION_ID_PROP) + ':' + musicbrainz_id + ']'
print 'query wikidata composition:', query
wikidata_composition_response = common.process_http_query(query)
print 'response wikidata composition content:', wikidata_composition_response.content
return wikidata_composition_response
def retrieve_viaf_compositions_by_id(viaf_id):
query_composition = VIAF_API_URL + str(viaf_id) + '/viaf.xml'
print 'query composition:', query_composition
composition_response = common.process_http_query(query_composition)
print 'viaf composition:', composition_response
if composition_response.content:
common.write_xml_file(VIAF_COMPOSITION_DIR, str(viaf_id), composition_response.content)
return composition_response
def retrieve_viaf_compositions_by_author_id(author_name, viaf_id, outputfile):
query_author = VIAF_API_URL + str(viaf_id) + '/viaf.xml'
print 'query author:', query_author
author_response = common.process_http_query(query_author)
print 'viaf author:', author_response
if author_response.content:
root = ET.fromstring(author_response.content)
parse_response(author_name, viaf_id, root, outputfile)
common.write_xml_file(VIAF_AUTHOR_DIR, str(viaf_id), author_response.content)
return author_response
def retrieve_wikidata_composition_viaf_id_by_wikidata_id(
wikidata_composition_id):
composition_response_json = None
try:
query_composition = WIKIDATA_API_URL + ITEMS_JSON + '[' + str(wikidata_composition_id) + ']&' + \
PROPS_JSON + '=' + str(VIAF_ID_PROP)
print 'query composition:', query_composition
composition_response_json = common.process_http_query(
query_composition)
except:
print 'No VIAF id found for composition ID:', wikidata_composition_id
print 'composition json data:', composition_response_json
return composition_response_json
def retrieve_musicbrainz_works_and_recordings_by_id(id, author, output_works, output_recordings):
try:
query_work = MUSICBRAINZ_API_URL + 'artist/' + id + '?inc=aliases%20works%20recordings&fmt=json'
# query_work = MUSICBRAINZ_API_URL + 'artist/' + id + '?inc=aliases%20works%20recordings&client=apikey&fmt=json'
#http://api.acoustid.org/v2/lookup?client=ULjKruIg&meta=recordings+releasegroups+compress&duration=641&fingerprint=AQABz0qUkZK4oOfhL-CPc4e5C_wW2H2QH9uDL4cvoT8UNQ-eHtsE8cceeFJx-LiiHT-aPzhxoc-Opj_eI5d2hOFyMJRzfDk-QSsu7fBxqZDMHcfxPfDIoPWxv9C1o3y
query_work = MUSICBRAINZ_API_URL + 'artist/' + id + '?inc=aliases%20works%20recordings&client=apikey&fmt=json'
print 'query work:', query_work
work_response = common.process_http_query(query_work)
print 'musicbrainz composition:', work_response
musicbrainz_composition_response_json = json.loads(work_response.content)
retrieve_compositions(musicbrainz_composition_response_json[common.WORKS_JSON], author, output_works, MUSICBRAINZ_WORKS_DIR)
retrieve_compositions(musicbrainz_composition_response_json[common.RECORDINGS_JSON], author, output_recordings, MUSICBRAINZ_RECORDINGS_DIR)
except ValueError as ve:
print 'Could not find JSON for given Musicbrainz composition.', id, ve.message
except Exception as e:
print 'Could not find Musicbrainz composition.', id, e.message
return work_response
def retrieve_musicbrainz_compositions_by_title(composition_title, viaf_id):
try:
query_work = MUSICBRAINZ_API_URL + 'work/?query=' + composition_title + '&fmt=json'
print 'query work:', query_work
work_response = common.process_http_query(query_work)
print 'musicbrainz composition:', work_response
musicbrainz_composition_response_json = json.loads(work_response.content)
works = musicbrainz_composition_response_json[common.WORKS_JSON]
if len(works) > 0:
json_data = works[0]
musicbrainz_composition_id = json_data[common.ID_JSON]
if str(musicbrainz_composition_id) + common.JSON_EXT not in os.listdir(MUSICBRAINZ_COMPOSITION_DIR):
print 'musicbrainz_composition_id:', musicbrainz_composition_id
store_musicbrainz_composition_data(musicbrainz_composition_id, json_data, MUSICBRAINZ_COMPOSITION_DIR)
store_mapping_composition_viafid_musicbranzid(viaf_id, musicbrainz_composition_id)
except ValueError as ve:
print 'Could not find JSON for given Musicbrainz composition.', composition_title, ve.message
except Exception as e:
print 'Could not find Musicbrainz composition.', composition_title, e.message
return work_response
def search_europeana_facets(url):
print 'search Europeana facets - url:', url
europeana_response = common.process_http_query(url)
print 'response content:', europeana_response.content
return europeana_response
