def _update_names(self):
prefix = settings.name_prefix.capitalize()
if self.outer_class is None:
if self.foreign_cpp_namespace:
self.full_name = self.foreign_cpp_namespace + '::' + self.name
else:
if self._module.cpp_namespace_prefix:
if self._module.cpp_namespace_prefix == '::':
self.full_name = '::' + self.name
else:
self.full_name = self._module.cpp_namespace_prefix + '::' + self.name
else:
self.full_name = self.name
else:
assert not self.foreign_cpp_namespace
self.full_name = '::'.join([self.outer_class.full_name, self.name])
def make_upper(s):
if s and s[0].islower():
return s[0].upper()+s[1:]
else:
return s
def flatten(name):
"make a name like::This look LikeThis"
return ''.join([make_upper(utils.mangle_name(s)) for s in name.split('::')])
self.mangled_name = flatten(self.name)
self.mangled_full_name = utils.mangle_name(self.full_name)
self.pytypestruct = "Py%s%s_Type" % (prefix, self.mangled_full_name)
def _update_names(self):
prefix = settings.name_prefix.capitalize()
if self.outer_class is None:
if self.foreign_cpp_namespace:
self.full_name = self.foreign_cpp_namespace + '::' + self.name
else:
if self._module.cpp_namespace_prefix:
if self._module.cpp_namespace_prefix == '::':
self.full_name = '::' + self.name
else:
self.full_name = self._module.cpp_namespace_prefix + '::' + self.name
else:
self.full_name = self.name
else:
assert not self.foreign_cpp_namespace
self.full_name = '::'.join([self.outer_class.full_name, self.name])
def make_upper(s):
if s and s[0].islower():
return s[0].upper() + s[1:]
else:
return s
def flatten(name):
"make a name like::This look LikeThis"
return ''.join(
[make_upper(utils.mangle_name(s)) for s in name.split('::')])
self.mangled_name = flatten(self.name)
self.mangled_full_name = utils.mangle_name(self.full_name)
self.pytypestruct = "Py%s%s_Type" % (prefix, self.mangled_full_name)
def are_similar(name1, name2):
name1, name2 = (mangle_name(s) for s in (name1, name2))
ratio = Levenshtein.jaro_winkler(name1, name2)
# TODO: remove this debug print
if ratio < 0.8:
print " * ratio = %s => name1 = '%s' vs name2 = '%s'" % (ratio, name1, name2)
return ratio >= 0.8
def are_similar(name1, name2):
name1, name2 = (mangle_name(s) for s in (name1, name2))
ratio = Levenshtein.jaro_winkler(name1, name2)
# TODO: remove this debug print
if ratio < 0.8:
print " * ratio = %s => name1 = '%s' vs name2 = '%s'" % (ratio, name1,
name2)
return ratio >= 0.8
def get_c_to_python_type_converter_function_name(self, value_type):
"""
Internal API, do not use.
"""
assert isinstance(value_type, TypeHandler)
ctype = value_type.ctype
mangled_ctype = utils.mangle_name(ctype)
converter_function_name = "_wrap_convert_c2py__%s" % mangled_ctype
return converter_function_name
def get_c_to_python_type_converter_function_name(self, value_type):
"""
Internal API, do not use.
"""
assert isinstance(value_type, TypeHandler)
ctype = value_type.ctype
mangled_ctype = utils.mangle_name(ctype)
converter_function_name = "_wrap_convert_c2py__%s" % mangled_ctype
return converter_function_name
def _update_names(self):
prefix = settings.name_prefix.capitalize()
if not self._full_name_is_definitive:
if self.outer_class is None:
if self._module.cpp_namespace_prefix:
if self._module.cpp_namespace_prefix == '::':
self.full_name = '::' + self.name
else:
self.full_name = self._module.cpp_namespace_prefix + '::' + self.name
else:
self.full_name = self.name
else:
self.full_name = '::'.join(
[self.outer_class.full_name, self.name])
def make_upper(s):
if s and s[0].islower():
return s[0].upper() + s[1:]
else:
return s
def flatten(name):
"make a name like::This look LikeThis"
return ''.join(
[make_upper(utils.mangle_name(s)) for s in name.split('::')])
self.mangled_name = flatten(self.name)
self.mangled_full_name = utils.mangle_name(self.full_name)
self._pystruct = "Py%s%s" % (prefix, self.mangled_full_name)
self.pytypestruct = "Py%s%s_Type" % (prefix, self.mangled_full_name)
self._iter_pystruct = "Py%s%sIter" % (prefix, self.mangled_full_name)
self.iter_pytypestruct = "Py%s%sIter_Type" % (prefix,
self.mangled_full_name)
## re-register the class type handlers, now with class full name
self.register_alias(self.full_name)
self.python_to_c_converter = self.module.get_root(
).get_python_to_c_type_converter_function_name(
self.ThisContainerReturn(self.full_name))
def _update_names(self):
prefix = settings.name_prefix.capitalize()
if not self._full_name_is_definitive:
if self.outer_class is None:
if self._module.cpp_namespace_prefix:
if self._module.cpp_namespace_prefix == '::':
self.full_name = '::' + self.name
else:
self.full_name = self._module.cpp_namespace_prefix + '::' + self.name
else:
self.full_name = self.name
else:
self.full_name = '::'.join([self.outer_class.full_name, self.name])
def make_upper(s):
if s and s[0].islower():
return s[0].upper()+s[1:]
else:
return s
def flatten(name):
"make a name like::This look LikeThis"
return ''.join([make_upper(utils.mangle_name(s)) for s in name.split('::')])
self.mangled_name = flatten(self.name)
self.mangled_full_name = utils.mangle_name(self.full_name)
self._pystruct = "Py%s%s" % (prefix, self.mangled_full_name)
self.pytypestruct = "Py%s%s_Type" % (prefix, self.mangled_full_name)
self._iter_pystruct = "Py%s%sIter" % (prefix, self.mangled_full_name)
self.iter_pytypestruct = "Py%s%sIter_Type" % (prefix, self.mangled_full_name)
## re-register the class type handlers, now with class full name
self.register_alias(self.full_name)
self.python_to_c_converter = self.module.get_root().get_python_to_c_type_converter_function_name(
self.ThisContainerReturn(self.full_name))
def name_to_mangled(name):
from utils import mangle_name
return mangle_name(name).upper()
query_label_artists = """
SELECT a.name
FROM s_artist a
JOIN artist_credit_name acn ON a.id = acn.artist
JOIN release r ON r.artist_credit = acn.artist_credit
JOIN release_label rl ON rl.release = r.id
WHERE rl.label = %s
"""
for id, gid, name in db.execute(query):
print 'Looking up label "%s" http://musicbrainz.org/label/%s' % (name, gid)
matches = wps.query(name.lower(), defType="dismax", qf="name", rows=50).results
last_wp_request = time.time()
for match in matches:
page_title = match["name"]
if mangle_name(re.sub(" \(.+\)$", "", page_title)) != mangle_name(name) and mangle_name(
page_title
) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({"action": "query", "prop": "revisions", "titles": page_title, "rvprop": "content"})
pages = resp["query"]["pages"].values()
if not pages or "revisions" not in pages[0]:
continue
page = mangle_name(pages[0]["revisions"][0].values()[0])
if "disambiguationpages" in page:
print " * disambiguation or album page, skipping"
continue
def __init__(self,
name,
value_type,
container_type,
outer_class=None,
custom_name=None):
"""
:param name: C++ type name of the container, e.g. std::vector<int> or MyIntList
:param value_type: a ReturnValue of the element type: note,
for mapping containers, value_type is a tuple with two
ReturnValue's: (key, element).
:param container_type: a string with the type of container,
one of 'list', 'deque', 'queue', 'priority_queue',
'vector', 'stack', 'set', 'multiset', 'hash_set',
'hash_multiset', 'map'
:param outer_class: if the type is defined inside a class, must be a reference to the outer class
:type outer_class: None or L{CppClass}
:param custom_name: alternative name to register with in the Python module
"""
if '<' in name or '::' in name:
self.name = utils.mangle_name(name)
self.full_name = name
self._full_name_is_definitive = True
else:
self._full_name_is_definitive = False
self.full_name = None
self.name = name
self._module = None
self.outer_class = outer_class
self.mangled_name = None
self.mangled_full_name = None
self.container_traits = container_traits_list[container_type]
self.custom_name = custom_name
self._pystruct = None
self.pytypestruct = "***GIVE ME A NAME***"
self.pytype = PyTypeObject()
self.iter_pytypestruct = "***GIVE ME A NAME***"
self.iter_pytype = PyTypeObject()
self._iter_pystruct = None
if self.container_traits.is_mapping:
(key_type, value_type) = value_type
self.key_type = utils.eval_retval(key_type, self)
self.value_type = utils.eval_retval(value_type, self)
else:
self.key_type = None
self.value_type = utils.eval_retval(value_type, self)
self.python_to_c_converter = None
if name != 'dummy':
## register type handlers
class ThisContainerParameter(ContainerParameter):
"""Register this C++ container as pass-by-value parameter"""
CTYPES = []
container_type = self
self.ThisContainerParameter = ThisContainerParameter
try:
param_type_matcher.register(name, self.ThisContainerParameter)
except ValueError:
pass
class ThisContainerRefParameter(ContainerRefParameter):
"""Register this C++ container as pass-by-value parameter"""
CTYPES = []
container_type = self
self.ThisContainerRefParameter = ThisContainerRefParameter
try:
param_type_matcher.register(name + '&',
self.ThisContainerRefParameter)
except ValueError:
pass
class ThisContainerPtrParameter(ContainerPtrParameter):
"""Register this C++ container as pass-by-ptr parameter"""
CTYPES = []
container_type = self
self.ThisContainerPtrParameter = ThisContainerPtrParameter
try:
param_type_matcher.register(name + '*',
self.ThisContainerPtrParameter)
except ValueError:
pass
class ThisContainerReturn(ContainerReturnValue):
"""Register this C++ container as value return"""
CTYPES = []
container_type = self
self.ThisContainerReturn = ThisContainerReturn
self.ThisContainerRefReturn = ThisContainerReturn
try:
return_type_matcher.register(name, self.ThisContainerReturn)
return_type_matcher.register(name, self.ThisContainerRefReturn)
except ValueError:
pass
category_re['fr'] = re.compile(r'\[\[Cat\xe9gorie:(.+?)\]\]')
for rg_id, rg_gid, rg_name, ac_name, rg_sec_types, processed in db.execute(
query, query_params):
colored_out(
bcolors.OKBLUE,
'Looking up release group "%s" http://musicbrainz.org/release-group/%s'
% (rg_name, rg_gid))
matches = wps.query(escape_query(rg_name),
defType='dismax',
qf='name',
rows=100).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(
rg_name) and mangle_name(title) != mangle_name(rg_name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
page_orig = get_page_content(wp, title, wp_lang)
if not page_orig:
continue
page_title = title
url = 'http://%s.wikipedia.org/wiki/%s' % (
wp_lang,
quote_page_title(page_title),
)
colored_out(bcolors.HEADER, ' * trying article %s' % (title, ))
if 'originalCredits' in shs_work and len(shs_work['originalCredits']) > 0:
shs_artists.extend(shs_work['originalCredits'])
# performer of original recording (bands are often wrongly credited as composer/writer in MB)
if 'original' in shs_work and shs_work['original'] is not None:
m = re.match(r'http://www.secondhandsongs.com/performance/([0-9]+)', shs_work['original']['uri'])
if m:
try:
original = shs.lookup('recording', int(m.group(1)))
if 'performer' in original:
shs_artists.append(original['performer']['artist'])
except ValueError:
pass
except urllib2.HTTPError:
pass
for shs_artist in shs_artists:
shs_artist_name = mangle_name(re.sub(' \[\d+\]$', '', shs_artist['commonName']))
mb_artist_name = mangle_name(artist['name'])
if shs_artist_name == mb_artist_name:
artist_uri = shs_artist['uri']
break
elif similarity2(to_unicode(shs_artist_name), to_unicode(mb_artist_name)) > 0.85:
print "%s => similarity = %.2f" % (shs_artist['commonName'], similarity2(to_unicode(shs_artist_name), to_unicode(mb_artist_name)))
artist_uri = shs_artist['uri']
break
if artist_uri:
matched_artists.add(artist['gid'])
colored_out(bcolors.HEADER, ' * using %s, found artist SHS URL: %s' % (artist['shs_url'], artist_uri))
edit_note = 'Guessing artist SecondHandSongs URL from work http://musicbrainz.org/work/%s linked to %s' % (artist['work_gid'], artist['shs_url'])
out(' * edit note: %s' % (edit_note,))
processed = 0
skipped = 0
for id, gid, name in db.execute(query):
processed += 1
if not contains_text_in_script(name, ['Hangul']):
skipped += 1
db.execute("INSERT INTO bot_wp_artist_ko (gid) VALUES (%s)", (gid,))
continue
print 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (name, gid)
matches = wps.query(name, defType='dismax', qf='name', rows=50).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if title.endswith('album)') or title.endswith('song)'):
continue
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(name) and mangle_name(title) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({'action': 'query', 'prop': 'revisions', 'titles': title, 'rvprop': 'content'})
pages = resp['query']['pages'].values()
if not pages or 'revisions' not in pages[0]:
continue
page = mangle_name(pages[0]['revisions'][0].values()[0])
if u'동음이의' in page:
print ' * disambiguation, skipping'
continue
print ' * trying article "%s"' % (title,)
page_title = pages[0]['title']
query_label_artists = """
SELECT a.name
FROM s_artist a
JOIN artist_credit_name acn ON a.id = acn.artist
JOIN release r ON r.artist_credit = acn.artist_credit
JOIN release_label rl ON rl.release = r.id
WHERE rl.label = %s
"""
for id, gid, name in db.execute(query):
print 'Looking up label "%s" http://musicbrainz.org/label/%s' % (name, gid)
matches = wps.query(name.lower(), defType='dismax', qf='name', rows=50).results
last_wp_request = time.time()
for match in matches:
page_title = match['name']
if mangle_name(re.sub(' \(.+\)$', '', page_title)) != mangle_name(name) and mangle_name(page_title) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({'action': 'query', 'prop': 'revisions', 'titles': page_title, 'rvprop': 'content'})
pages = resp['query']['pages'].values()
if not pages or 'revisions' not in pages[0]:
continue
page = mangle_name(pages[0]['revisions'][0].values()[0])
if 'disambiguationpages' in page:
print ' * disambiguation or album page, skipping'
continue
if 'recordlabels' not in page:
print ' * not a record label page, skipping'
def flatten(name):
"make a name like::This look LikeThis"
return ''.join([make_upper(utils.mangle_name(s)) for s in name.split('::')])
def __init__(self, name, value_type, container_type, outer_class=None, custom_name=None):
"""
:param name: C++ type name of the container, e.g. std::vector<int> or MyIntList
:param value_type: a ReturnValue of the element type: note,
for mapping containers, value_type is a tuple with two
ReturnValue's: (key, element).
:param container_type: a string with the type of container,
one of 'list', 'deque', 'queue', 'priority_queue',
'vector', 'stack', 'set', 'multiset', 'hash_set',
'hash_multiset', 'map'
:param outer_class: if the type is defined inside a class, must be a reference to the outer class
:type outer_class: None or L{CppClass}
:param custom_name: alternative name to register with in the Python module
"""
if '<' in name or '::' in name:
self.name = utils.mangle_name(name)
self.full_name = name
self._full_name_is_definitive = True
else:
self._full_name_is_definitive = False
self.full_name = None
self.name = name
self._module = None
self.outer_class = outer_class
self.mangled_name = None
self.mangled_full_name = None
self.container_traits = container_traits_list[container_type]
self.custom_name = custom_name
self._pystruct = None
self.pytypestruct = "***GIVE ME A NAME***"
self.pytype = PyTypeObject()
self.iter_pytypestruct = "***GIVE ME A NAME***"
self.iter_pytype = PyTypeObject()
self._iter_pystruct = None
if self.container_traits.is_mapping:
(key_type, value_type) = value_type
self.key_type = utils.eval_retval(key_type, self)
self.value_type = utils.eval_retval(value_type, self)
else:
self.key_type = None
self.value_type = utils.eval_retval(value_type, self)
self.python_to_c_converter = None
if name != 'dummy':
## register type handlers
class ThisContainerParameter(ContainerParameter):
"""Register this C++ container as pass-by-value parameter"""
CTYPES = []
container_type = self
self.ThisContainerParameter = ThisContainerParameter
try:
param_type_matcher.register(name, self.ThisContainerParameter)
except ValueError:
pass
class ThisContainerRefParameter(ContainerRefParameter):
"""Register this C++ container as pass-by-value parameter"""
CTYPES = []
container_type = self
self.ThisContainerRefParameter = ThisContainerRefParameter
try:
param_type_matcher.register(name+'&', self.ThisContainerRefParameter)
except ValueError:
pass
class ThisContainerPtrParameter(ContainerPtrParameter):
"""Register this C++ container as pass-by-ptr parameter"""
CTYPES = []
container_type = self
self.ThisContainerPtrParameter = ThisContainerPtrParameter
try:
param_type_matcher.register(name+'*', self.ThisContainerPtrParameter)
except ValueError:
pass
class ThisContainerReturn(ContainerReturnValue):
"""Register this C++ container as value return"""
CTYPES = []
container_type = self
self.ThisContainerReturn = ThisContainerReturn
self.ThisContainerRefReturn = ThisContainerReturn
try:
return_type_matcher.register(name, self.ThisContainerReturn)
return_type_matcher.register(name, self.ThisContainerRefReturn)
except ValueError:
pass
processed = 0
skipped = 0
for id, gid, name in db.execute(query):
processed += 1
if not contains_text_in_script(name, ["Katakana", "Hiragana", "Han"]):
skipped += 1
db.execute("INSERT INTO bot_wp_artist_ja (gid) VALUES (%s)", (gid,))
continue
print 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (name, gid)
matches = wps.query(name, defType="dismax", qf="name", rows=50).results
last_wp_request = time.time()
for match in matches:
title = match["name"]
if title.endswith("album)") or title.endswith("song)"):
continue
if mangle_name(re.sub(" \(.+\)$", "", title)) != mangle_name(name) and mangle_name(title) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({"action": "query", "prop": "revisions", "titles": title, "rvprop": "content"})
pages = resp["query"]["pages"].values()
if not pages or "revisions" not in pages[0]:
continue
page = mangle_name(pages[0]["revisions"][0].values()[0])
if u"曖昧さ回避" in page:
print " * disambiguation, skipping"
continue
print ' * trying article "%s"' % (title,)
page_title = pages[0]["title"]
processed = 0
skipped = 0
for id, gid, name in db.execute(query):
processed += 1
if not contains_text_in_script(name, ['Hangul']):
skipped += 1
db.execute("INSERT INTO bot_wp_artist_ko (gid) VALUES (%s)", (gid,))
continue
print 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (name, gid)
matches = wps.query(name, defType='dismax', qf='name', rows=50).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if title.endswith('album)') or title.endswith('song)'):
continue
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(name) and mangle_name(title) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({'action': 'query', 'prop': 'revisions', 'titles': title, 'rvprop': 'content'})
pages = resp['query']['pages'].values()
if not pages or 'revisions' not in pages[0]:
continue
page = mangle_name(pages[0]['revisions'][0].values()[0])
if u'동음이의' in page:
print ' * disambiguation, skipping'
continue
print ' * trying article "%s"' % (title,)
page_title = pages[0]['title']
def flatten(name):
"make a name like::This look LikeThis"
return ''.join(
[make_upper(utils.mangle_name(s)) for s in name.split('::')])
UNION
SELECT r.name
FROM s_release r
JOIN artist_credit_name acn ON r.artist_credit = acn.artist_credit
WHERE acn.artist = %s
"""
for artist in db.execute(query, query_params):
colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))
matches = wps.query(escape_query(artist['name']), defType='dismax', qf='name', rows=50).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if title.endswith('album)') or title.endswith('song)'):
continue
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(artist['name']) and mangle_name(title) != mangle_name(artist['name']):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
wikipage = WikiPage.fetch('http://%s.wikipedia.org/wiki/%s' % (wp_lang, title))
page_orig = wikipage.text
if not page_orig:
continue
out(' * trying article "%s"' % (title,))
page = mangle_name(page_orig)
if 'redirect' in page:
out(' * redirect page, skipping')
continue
if 'disambiguation' in title:
JOIN recording ON acn.artist_credit = recording.artist_credit
JOIN l_recording_work lrw ON recording.id = lrw.entity0
JOIN l_artist_work law ON lrw.entity1 = law.entity1
WHERE law.entity0 = %s
)
"""
for artist in db.execute(query, query_params):
colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))
matches = wps.query(escape_query(artist['name']), defType='dismax', qf='name', rows=50).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if title.endswith('album)') or title.endswith('song)'):
continue
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(artist['name']) and mangle_name(title) != mangle_name(artist['name']):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
wikipage = WikiPage.fetch('https://%s.wikipedia.org/wiki/%s' % (wp_lang, title))
page_orig = wikipage.text
if not page_orig:
continue
out(' * trying article "%s"' % (title,))
page = mangle_name(page_orig)
is_canonical, reason = wp_is_canonical_page(title, page_orig)
if (not is_canonical):
out(' * %s, skipping' % reason)
JOIN medium m ON t.medium=m.id
JOIN release r ON m.release=r.id
WHERE r.release_group = %s
"""
category_re = {}
category_re['en'] = re.compile(r'\[\[Category:(.+?)(?:\|.*?)?\]\]')
category_re['fr'] = re.compile(r'\[\[Cat\xe9gorie:(.+?)\]\]')
for rg_id, rg_gid, rg_name, ac_name, rg_sec_types, processed in db.execute(query, query_params):
colored_out(bcolors.OKBLUE, 'Looking up release group "%s" https://musicbrainz.org/release-group/%s' % (rg_name, rg_gid))
matches = wps.query(escape_query(rg_name), defType='dismax', qf='name', rows=100).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(rg_name) and mangle_name(title) != mangle_name(rg_name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
wikipage = WikiPage.fetch('https://%s.wikipedia.org/wiki/%s' % (wp_lang, title))
page_orig = wikipage.text
if not page_orig:
continue
page_title = title
colored_out(bcolors.HEADER, ' * trying article %s' % (title,))
page = mangle_name(page_orig)
is_canonical, reason = wp_is_canonical_page(title, page_orig)
if (not is_canonical):
for artist in db.execute(query, query_params):
colored_out(
bcolors.OKBLUE,
'Looking up artist "%s" http://musicbrainz.org/artist/%s' %
(artist['name'], artist['gid']))
matches = wps.query(escape_query(artist['name']),
defType='dismax',
qf='name',
rows=50).results
last_wp_request = time.time()
for match in matches:
title = match['name']
if title.endswith('album)') or title.endswith('song)'):
continue
if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(
artist['name']) and mangle_name(title) != mangle_name(
artist['name']):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
wikipage = WikiPage.fetch('http://%s.wikipedia.org/wiki/%s' %
(wp_lang, title))
page_orig = wikipage.text
if not page_orig:
continue
out(' * trying article "%s"' % (title, ))
page = mangle_name(page_orig)
FROM s_artist a
JOIN artist_credit_name acn ON a.id = acn.artist
JOIN release r ON r.artist_credit = acn.artist_credit
JOIN release_label rl ON rl.release = r.id
WHERE rl.label = %s
"""
for id, gid, name in db.execute(query):
print 'Looking up label "%s" https://musicbrainz.org/label/%s' % (name,
gid)
matches = wps.query(name.lower(), defType='dismax', qf='name',
rows=50).results
last_wp_request = time.time()
for match in matches:
page_title = match['name']
if mangle_name(re.sub(' \(.+\)$', '', page_title)) != mangle_name(
name) and mangle_name(page_title) != mangle_name(name):
continue
delay = time.time() - last_wp_request
if delay < 1.0:
time.sleep(1.0 - delay)
last_wp_request = time.time()
resp = wp.call({
'action': 'query',
'prop': 'revisions',
'titles': page_title,
'rvprop': 'content'
})
pages = resp['query']['pages'].values()
if not pages or 'revisions' not in pages[0]:
continue
if 'originalCredits' in shs_work and len(shs_work['originalCredits']) > 0:
shs_artists.extend(shs_work['originalCredits'])
# performer of original recording (bands are often wrongly credited as composer/writer in MB)
if 'original' in shs_work and shs_work['original'] is not None:
m = re.match(r'http://www.secondhandsongs.com/performance/([0-9]+)', shs_work['original']['uri'])
if m:
try:
original = shs.lookup('recording', int(m.group(1)))
if 'performer' in original:
shs_artists.append(original['performer']['artist'])
except ValueError:
pass
except urllib2.HTTPError:
pass
for shs_artist in shs_artists:
shs_artist_name = mangle_name(re.sub(' \[\d+\]$', '', shs_artist['commonName']))
mb_artist_name = mangle_name(artist['name'])
if shs_artist_name == mb_artist_name:
artist_uri = shs_artist['uri']
break
elif similarity2(to_unicode(shs_artist_name), to_unicode(mb_artist_name)) > 0.85:
print " * '%s' has a similarity of %.2f" % (shs_artist['commonName'], similarity2(to_unicode(shs_artist_name), to_unicode(mb_artist_name)))
artist_uri = shs_artist['uri']
break
if artist_uri:
matched_artists.add(artist['gid'])
colored_out(bcolors.HEADER, ' * using %s, found artist SHS URL: %s' % (artist['shs_url'], artist_uri))
edit_note = 'Guessing artist SecondHandSongs URL from work https://musicbrainz.org/work/%s linked to %s' % (artist['work_gid'], artist['shs_url'])
out(' * edit note: %s' % (edit_note,))
