def find_song_info(self, content):
# content = content.decode('utf-8', 'ignore')
pattern = u'<meta property="og:title" content="(.*?) 歌詞 \|'
title = common.get_first_group_by_pattern(content, pattern)
title = common.htmlspecialchars_decode(title)
self.title = title
pattern = u'<meta property="og:description" content="(.*?)が歌う'
artist = common.get_first_group_by_pattern(content, pattern)
artist = common.htmlspecialchars_decode(artist)
self.artist = artist
prefixes = {
'lyricist': u'作詞</dt>',
'composer': u'作曲</dt>',
}
suffix = '</dd>'
for key in prefixes:
prefix = prefixes[key]
value = common.find_string_by_prefix_suffix(content, prefix, suffix, False)
if value:
value = common.strip_tags(value).strip()
value = common.htmlspecialchars_decode(value)
setattr(self, key, value)
return True
def find_song_info(self, content):
prefix = "<div id='lyricBlock'>"
suffix = '</table>'
info_block = common.find_string_by_prefix_suffix(content, prefix, suffix, False)
prefix = '<h2>'
suffix = '</h2>'
title = common.find_string_by_prefix_suffix(info_block, prefix, suffix, False)
self.title = common.htmlspecialchars_decode(common.unicode2string(title))
patterns = {
'artist': u'>歌：(.*?)</td>',
'lyricist': u'>作詞：(.*?)</td>',
'composer': u'>作曲：(.*?)</td>'
}
for key in patterns:
pattern = patterns[key]
value = common.get_first_group_by_pattern(info_block, pattern)
if value:
value = common.strip_tags(common.htmlspecialchars_decode(value)).strip()
setattr(self, key, value)
else:
logging.debug('Failed to get %s, pattern: %s' % (key, pattern, ))
return True
def parse_song_info(self, url, html):
ret = True
pattern = '<h1.*?>(.*)</h1>'
title = common.get_first_group_by_pattern(html, pattern)
if title:
title = common.htmlspecialchars_decode(common.strip_tags(title)).strip()
self.title = title
else:
logging.warning('Failed to parse title of url [%s]', url)
ret = False
patterns = {
'artist': '">([^<]*)</a></span></td></tr>',
'lyricist': 'td>([^<]*)<br></td></tr><tr>',
'composer': 'td>([^<]*)<br></td></tr></table>',
}
prefix = '<table class="lyric-data">'
suffix = '</table>'
info_table = common.find_string_by_prefix_suffix(html, prefix, suffix)
info_table = info_table.replace('\n', '')
for key in patterns:
pattern = patterns[key]
value = common.get_first_group_by_pattern(info_table, pattern)
if value:
value = common.htmlspecialchars_decode(common.strip_tags(value)).strip()
setattr(self, key, value)
else:
logging.warning('Failed to get %s', key)
ret = False
return ret
def find_lyric(self, url):
pattern = 'item-([0-9]+)\.html'
song_id = common.get_first_group_by_pattern(url, pattern)
if not song_id:
logging.info('Failed to get id of url [%s]', url)
return False
song_url = 'http://www.kasi-time.com/item_js.php?no=' + song_id
data = common.get_url_content(song_url)
if not data:
logging.info('Failed to get content of url [%s]', song_url)
return False
lyric = data.decode('utf-8', 'ignore')
lyric = lyric.replace("document.write('", "")
lyric = lyric.replace("');", "")
lyric = lyric.replace("<br>", "\n")
lyric = lyric.replace(" ", " ")
lyric = common.htmlspecialchars_decode(lyric)
lyric = common.unicode2string(lyric)
lyric = common.strip_slash(lyric)
lyric = lyric.strip()
# test for half to full
lyric = common.half2full(lyric)
self.lyric = lyric
return True
def find_song_info(self, url):
ret = True
html = common.get_url_content(url)
encoding = 'euc_jp'
html = html.decode(encoding, 'ignore')
patterns = {
'title': 'title',
'artist': 'artist',
'lyricist': 'sakusi',
'composer': 'sakyoku',
}
for key in patterns:
key_for_pattern = patterns[key]
pattern = u'<INPUT type="hidden" name=%s value="([^"]*)">' % (key_for_pattern, )
value = common.get_first_group_by_pattern(html, pattern)
if not value:
logging.info('Failed to get %s of url [%s]', key, url)
ret = False
else:
value = common.htmlspecialchars_decode(value).strip()
setattr(self, key, value)
return ret
def parse_composer(self, html):
prefix = '<b>作曲：</b>'
suffix = '\t'
raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
if not raw_string:
logging.debug('Failed to find composer')
return False
self.composer = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()
def find_lyric(self, content):
prefix = "<p id='lyricBody'>"
suffix = "</p>"
lyric = common.find_string_by_prefix_suffix(content, prefix, suffix, False)
lyric = lyric.replace('<br />', '')
lyric = common.htmlspecialchars_decode(common.unicode2string(lyric))
lyric = lyric.strip();
self.lyric = lyric
return True
def parse_lyricist(self, html):
prefix = '<b>作詞：</b>'
suffix = '\t'
logging.debug('find me LYRICIST')
raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
if not raw_string:
logging.debug('Failed to find lyricist')
return False
self.lyricist = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()
def parse_lyric(self, html):
prefix = '<PRE>'
suffix = '</PRE>'
raw_lyric = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
if not raw_lyric:
logging.info('Failed to parse lyric from html [%s]', html)
return False
raw_lyric = raw_lyric.strip()
pos = raw_lyric.find('\n\n')
if pos < 0:
logging.info('Failed to find two consecutive newlines')
return False
self.title = raw_lyric[0:pos]
raw_lyric = raw_lyric[pos+2:]
pos = raw_lyric.find('\n\n')
if pos < 0:
logging.info('Failed to find two consecutive newlines')
return False
info = raw_lyric[0:pos]
self.lyric = raw_lyric[pos+2:]
patterns = {
'artist': u'歌：(.+)',
'lyricist': u'作詞：(.+?)／',
'composer': u'作曲：(.+?)／',
'arranger': u'編曲：(.+?)／',
}
for key in patterns:
pattern = patterns[key]
value = common.get_first_group_by_pattern(info, pattern)
if not value:
logging.info('Failed to get %s of url [%s]', key, url)
else:
value = common.htmlspecialchars_decode(value).strip()
setattr(self, key, value)
return True
def find_song_info(self, url):
ret = True
html = common.get_url_content(url)
encoding = 'euc_jp'
html = html.decode(encoding, 'ignore')
prefix = '<TABLE cellspacing="1"'
suffix = '</TABLE>'
info_table = common.find_string_by_prefix_suffix(html, prefix, suffix)
def get_info_value_by_key(html, key):
valuePrefix = '#ffffff> '
valueSuffix = '</TD>'
lenPrefix = len(valuePrefix)
posKey = html.find(key)
logging.debug('key position: %d' % (posKey))
posStart = html.find(valuePrefix, posKey) + lenPrefix
posEnd = html.find(valueSuffix, posStart)
logging.debug('position [%d:%d]' % (posStart, posEnd))
value = html[posStart:posEnd]
return value
patterns = {
'title': u'曲名</TD>',
'artist': u'歌手</TD>',
'lyricist': u'作詞</TD>',
'composer': u'作曲</TD>',
}
for key in patterns:
pattern = patterns[key]
value = get_info_value_by_key(info_table, pattern)
if not value:
logging.info('Failed to get %s of url [%s]', key, url)
ret = False
else:
value = common.htmlspecialchars_decode(value).strip()
setattr(self, key, value)
return ret
def parse_page(self):
url = self.url
s = requests.Session()
self.s = s
# 1. get song id from url
id = self.get_song_id(url)
if not id:
logging.error('Failed to get id of url [%s]', url)
return False
r = s.get(url)
plsession = r.cookies['PLSESSION']
# 2. get 1st part lyric from html and song_info
html = r.text
if not html:
logging.info('Failed to get html of url [%s]' % (url))
return False
lyric_1st = self.get_lyric_1st_part(html)
# 2. get CSRF token
token = self.get_csrf_token(html)
if not token:
logging.info('Failed to get CSRF token of url [%s]' % (url))
return False
logging.debug('CSRF token is %s' % (token, ))
# 2. get second part lyric
lyric_2nd = self.get_lyric_2nd_part(id, token, plsession)
self.lyric = common.htmlspecialchars_decode(lyric_1st + lyric_2nd)
self.parse_artist_title(html)
self.parse_lyricist(html)
self.parse_composer(html)
return True
def find_song_info(self, doc):
ret = True
patterns = {
'title': 'Title',
'artist': 'ArtistName',
'lyricist': 'WriterName',
'composer': 'ComposerName',
}
for key in patterns:
tag = patterns[key]
value = self.xml_get_first_child(doc, tag)
if not value:
logging.info('Failed to get %s of url [%s]', key, url)
ret = False
else:
value = common.htmlspecialchars_decode(value).strip()
setattr(self, key, value)
return ret
def find_song_info(self, content):
pattern = 'og:description" content="(.*)"'
og_desc = common.get_first_group_by_pattern(content, pattern)
if og_desc:
pattern = u'(.*?)「(.*?)」'
matches = common.get_matches_by_pattern(og_desc, pattern)
if matches:
artist = matches.group(1)
artist = artist.replace(u'歌詞サーチ ', '')
self.artist = artist
self.title = matches.group(2)
else:
logging.debug('og desc: %s' % (og_desc))
prefix = '="lyrics_info_text"'
suffix = '</div>'
info_text = common.find_string_by_prefix_suffix(content, prefix, suffix, False)
if not info_text:
logging.info('Failed to find lyrics info text')
one_line = info_text.replace('\n', '')
patterns = {
'lyricist': u'>作詞</p><p class="info_detail">(.*?)</p>',
'composer': u'>作曲</p><p class="info_detail">(.*?)</p>',
}
for key in patterns:
pattern = patterns[key]
value = common.get_first_group_by_pattern(one_line, pattern)
if value:
value = common.strip_tags(common.htmlspecialchars_decode(value)).strip()
setattr(self, key, value)
else:
logging.debug('Failed to get %s, pattern: %s' % (key, pattern, ))
return True
def sanitize(self, src):
return common.unicode2string(common.htmlspecialchars_decode(src))
