def find_song_info(self, content): # content = content.decode('utf-8', 'ignore') pattern = u'<meta property="og:title" content="(.*?) 歌詞 \|' title = common.get_first_group_by_pattern(content, pattern) title = common.htmlspecialchars_decode(title) self.title = title pattern = u'<meta property="og:description" content="(.*?)が歌う' artist = common.get_first_group_by_pattern(content, pattern) artist = common.htmlspecialchars_decode(artist) self.artist = artist prefixes = { 'lyricist': u'作詞</dt>', 'composer': u'作曲</dt>', } suffix = '</dd>' for key in prefixes: prefix = prefixes[key] value = common.find_string_by_prefix_suffix(content, prefix, suffix, False) if value: value = common.strip_tags(value).strip() value = common.htmlspecialchars_decode(value) setattr(self, key, value) return True
def find_song_info(self, content): prefix = "<div id='lyricBlock'>" suffix = '</table>' info_block = common.find_string_by_prefix_suffix(content, prefix, suffix, False) prefix = '<h2>' suffix = '</h2>' title = common.find_string_by_prefix_suffix(info_block, prefix, suffix, False) self.title = common.htmlspecialchars_decode(common.unicode2string(title)) patterns = { 'artist': u'>歌:(.*?)</td>', 'lyricist': u'>作詞:(.*?)</td>', 'composer': u'>作曲:(.*?)</td>' } for key in patterns: pattern = patterns[key] value = common.get_first_group_by_pattern(info_block, pattern) if value: value = common.strip_tags(common.htmlspecialchars_decode(value)).strip() setattr(self, key, value) else: logging.debug('Failed to get %s, pattern: %s' % (key, pattern, )) return True
def parse_song_info(self, url, html): ret = True pattern = '<h1.*?>(.*)</h1>' title = common.get_first_group_by_pattern(html, pattern) if title: title = common.htmlspecialchars_decode(common.strip_tags(title)).strip() self.title = title else: logging.warning('Failed to parse title of url [%s]', url) ret = False patterns = { 'artist': '">([^<]*)</a></span></td></tr>', 'lyricist': 'td>([^<]*)<br></td></tr><tr>', 'composer': 'td>([^<]*)<br></td></tr></table>', } prefix = '<table class="lyric-data">' suffix = '</table>' info_table = common.find_string_by_prefix_suffix(html, prefix, suffix) info_table = info_table.replace('\n', '') for key in patterns: pattern = patterns[key] value = common.get_first_group_by_pattern(info_table, pattern) if value: value = common.htmlspecialchars_decode(common.strip_tags(value)).strip() setattr(self, key, value) else: logging.warning('Failed to get %s', key) ret = False return ret
def find_lyric(self, url): pattern = 'item-([0-9]+)\.html' song_id = common.get_first_group_by_pattern(url, pattern) if not song_id: logging.info('Failed to get id of url [%s]', url) return False song_url = 'http://www.kasi-time.com/item_js.php?no=' + song_id data = common.get_url_content(song_url) if not data: logging.info('Failed to get content of url [%s]', song_url) return False lyric = data.decode('utf-8', 'ignore') lyric = lyric.replace("document.write('", "") lyric = lyric.replace("');", "") lyric = lyric.replace("<br>", "\n") lyric = lyric.replace(" ", " ") lyric = common.htmlspecialchars_decode(lyric) lyric = common.unicode2string(lyric) lyric = common.strip_slash(lyric) lyric = lyric.strip() # test for half to full lyric = common.half2full(lyric) self.lyric = lyric return True
def find_song_info(self, url): ret = True html = common.get_url_content(url) encoding = 'euc_jp' html = html.decode(encoding, 'ignore') patterns = { 'title': 'title', 'artist': 'artist', 'lyricist': 'sakusi', 'composer': 'sakyoku', } for key in patterns: key_for_pattern = patterns[key] pattern = u'<INPUT type="hidden" name=%s value="([^"]*)">' % (key_for_pattern, ) value = common.get_first_group_by_pattern(html, pattern) if not value: logging.info('Failed to get %s of url [%s]', key, url) ret = False else: value = common.htmlspecialchars_decode(value).strip() setattr(self, key, value) return ret
def parse_composer(self, html): prefix = '<b>作曲:</b>' suffix = '\t' raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False) if not raw_string: logging.debug('Failed to find composer') return False self.composer = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()
def find_lyric(self, content): prefix = "<p id='lyricBody'>" suffix = "</p>" lyric = common.find_string_by_prefix_suffix(content, prefix, suffix, False) lyric = lyric.replace('<br />', '') lyric = common.htmlspecialchars_decode(common.unicode2string(lyric)) lyric = lyric.strip(); self.lyric = lyric return True
def parse_lyricist(self, html): prefix = '<b>作詞:</b>' suffix = '\t' logging.debug('find me LYRICIST') raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False) if not raw_string: logging.debug('Failed to find lyricist') return False self.lyricist = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()
def parse_lyric(self, html): prefix = '<PRE>' suffix = '</PRE>' raw_lyric = common.find_string_by_prefix_suffix(html, prefix, suffix, False) if not raw_lyric: logging.info('Failed to parse lyric from html [%s]', html) return False raw_lyric = raw_lyric.strip() pos = raw_lyric.find('\n\n') if pos < 0: logging.info('Failed to find two consecutive newlines') return False self.title = raw_lyric[0:pos] raw_lyric = raw_lyric[pos+2:] pos = raw_lyric.find('\n\n') if pos < 0: logging.info('Failed to find two consecutive newlines') return False info = raw_lyric[0:pos] self.lyric = raw_lyric[pos+2:] patterns = { 'artist': u'歌:(.+)', 'lyricist': u'作詞:(.+?)/', 'composer': u'作曲:(.+?)/', 'arranger': u'編曲:(.+?)/', } for key in patterns: pattern = patterns[key] value = common.get_first_group_by_pattern(info, pattern) if not value: logging.info('Failed to get %s of url [%s]', key, url) else: value = common.htmlspecialchars_decode(value).strip() setattr(self, key, value) return True
def find_song_info(self, url): ret = True html = common.get_url_content(url) encoding = 'euc_jp' html = html.decode(encoding, 'ignore') prefix = '<TABLE cellspacing="1"' suffix = '</TABLE>' info_table = common.find_string_by_prefix_suffix(html, prefix, suffix) def get_info_value_by_key(html, key): valuePrefix = '#ffffff> ' valueSuffix = '</TD>' lenPrefix = len(valuePrefix) posKey = html.find(key) logging.debug('key position: %d' % (posKey)) posStart = html.find(valuePrefix, posKey) + lenPrefix posEnd = html.find(valueSuffix, posStart) logging.debug('position [%d:%d]' % (posStart, posEnd)) value = html[posStart:posEnd] return value patterns = { 'title': u'曲名</TD>', 'artist': u'歌手</TD>', 'lyricist': u'作詞</TD>', 'composer': u'作曲</TD>', } for key in patterns: pattern = patterns[key] value = get_info_value_by_key(info_table, pattern) if not value: logging.info('Failed to get %s of url [%s]', key, url) ret = False else: value = common.htmlspecialchars_decode(value).strip() setattr(self, key, value) return ret
def parse_page(self): url = self.url s = requests.Session() self.s = s # 1. get song id from url id = self.get_song_id(url) if not id: logging.error('Failed to get id of url [%s]', url) return False r = s.get(url) plsession = r.cookies['PLSESSION'] # 2. get 1st part lyric from html and song_info html = r.text if not html: logging.info('Failed to get html of url [%s]' % (url)) return False lyric_1st = self.get_lyric_1st_part(html) # 2. get CSRF token token = self.get_csrf_token(html) if not token: logging.info('Failed to get CSRF token of url [%s]' % (url)) return False logging.debug('CSRF token is %s' % (token, )) # 2. get second part lyric lyric_2nd = self.get_lyric_2nd_part(id, token, plsession) self.lyric = common.htmlspecialchars_decode(lyric_1st + lyric_2nd) self.parse_artist_title(html) self.parse_lyricist(html) self.parse_composer(html) return True
def find_song_info(self, doc): ret = True patterns = { 'title': 'Title', 'artist': 'ArtistName', 'lyricist': 'WriterName', 'composer': 'ComposerName', } for key in patterns: tag = patterns[key] value = self.xml_get_first_child(doc, tag) if not value: logging.info('Failed to get %s of url [%s]', key, url) ret = False else: value = common.htmlspecialchars_decode(value).strip() setattr(self, key, value) return ret
def find_song_info(self, content): pattern = 'og:description" content="(.*)"' og_desc = common.get_first_group_by_pattern(content, pattern) if og_desc: pattern = u'(.*?)「(.*?)」' matches = common.get_matches_by_pattern(og_desc, pattern) if matches: artist = matches.group(1) artist = artist.replace(u'歌詞サーチ ', '') self.artist = artist self.title = matches.group(2) else: logging.debug('og desc: %s' % (og_desc)) prefix = '="lyrics_info_text"' suffix = '</div>' info_text = common.find_string_by_prefix_suffix(content, prefix, suffix, False) if not info_text: logging.info('Failed to find lyrics info text') one_line = info_text.replace('\n', '') patterns = { 'lyricist': u'>作詞</p><p class="info_detail">(.*?)</p>', 'composer': u'>作曲</p><p class="info_detail">(.*?)</p>', } for key in patterns: pattern = patterns[key] value = common.get_first_group_by_pattern(one_line, pattern) if value: value = common.strip_tags(common.htmlspecialchars_decode(value)).strip() setattr(self, key, value) else: logging.debug('Failed to get %s, pattern: %s' % (key, pattern, )) return True
def sanitize(self, src): return common.unicode2string(common.htmlspecialchars_decode(src))