def get_system_default_avatar_url():
"""
返回系统默认的头像url地址
:return:
"""
from conf.cm import ConfigManager
cm = ConfigManager()
avatar_url = "http://" + cm.get_config('host') + ":" + str(cm.get_config('port')) + cm.get_config('upload')['visit']['avatar'] + 'default.png'
return avatar_url
def __init__(self):
cm = ConfigManager()
db_config = cm.get_dbconfig()
self.conn_r = MySQLdb.connect(
host=db_config['host'],
user=db_config['user'],
passwd=db_config['passwd'],
db=db_config['db'],
charset=db_config['charset']
)
def save_file(file_name, file_content, dir_type):
"""
保存文件到本地
:param file_name: 文件名
:param file_content: 文件内容
:param dir_type: 保存路径类型 (1: avatar 2: question_pic 3: question_sound)
:return:
"""
from conf.cm import ConfigManager
cm = ConfigManager()
if dir_type == 1:
upload_dir = cm.get_config('upload')['save']['avatar']
url = cm.get_config('upload')['visit']['avatar']
elif dir_type == 2:
upload_dir = cm.get_config('upload')['save']['pic']
url = cm.get_config('upload')['visit']['pic']
elif dir_type == 3:
upload_dir = cm.get_config('upload')['save']['sound']
url = cm.get_config('upload')['visit']['sound']
output_file = open(upload_dir + file_name, 'wb')
output_file.write(file_content)
output_file.flush()
output_file.close()
return url + file_name
class DataProcedure():
def __init__(self):
self.cm = ConfigManager()
self.db = DBManager()
def __del__(self):
self.db.close()
def run(self):
sql = self.get_sql()
queue_data = self.cm.get_config('taskqueue')['data']
queue_back = self.cm.get_config('taskqueue')['backup']
r = redis.Redis(
host=self.cm.get_config('redis')['host'],
port=self.cm.get_config('redis')['port']
)
if not r:
print 'Redis服务未启动'
else:
print 'Redis服务正常'
# 处理当前任务
cur_task = r.rpoplpush(queue_data, queue_back)
while cur_task is not None:
# print cur_task
is_success, rows = self.db.save(sql, eval(cur_task))
if is_success:
# 提交成功, 清空备份队列
r.delete(queue_back)
cur_task = r.rpoplpush(queue_data, queue_back)
print '队列中没有要处理的任务'
def get_sql(self):
table_name = '%sresult' % self.cm.get_config('table')[0]['song']['prefix']
sql_data_save = ("INSERT INTO %s " % table_name) + "(`sid`, `author`, `sname`, `counts`, `durl`) " \
"VALUES (%s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE " \
"counts=counts+1;"
return sql_data_save
def __init__(self):
self.cm = ConfigManager()
self.db = DBManager()
class Baidu():
def __init__(self):
self.cm = ConfigManager()
self.db = DBManager()
def __del__(self):
self.db.close()
def get_song_nums(self, *args, **kwargs):
"""
获取实际歌曲数，存在同歌曲名，不同sid的情形，故需要去重
:param args: 查询的集合域
:param kwargs: 过滤条件，可以取值歌手，歌名或者其他
:return:
"""
table = '%sresult' % self.cm.get_config('table')[0]['song']['prefix']
category = ', '.join(args)
filter_key = kwargs.keys()[0]
filter_value = kwargs.get(filter_key)
sql_search = ('SELECT COUNT(DISTINCT %s) FROM %s ' % (category, table)) + 'WHERE %s = \'%s\'' % (filter_key,
filter_value)
data = self.db.query(sql_search)
return data[0][0]
def searchBySinger(self, singer):
"""
根据歌曲名称查询
优先搜索数据库, 若找到, 直接返回该数据
提示用户是否仍要继续下载
否则, 联网搜索, 并将新数据存入数据库
:param singer: 歌手
:return:
"""
table = '%sresult' % self.cm.get_config('table')[0]['song']['prefix']
sql_search = ('SELECT sname, durl FROM %s ' % table) + 'WHERE author = \'%s\'' % singer
data = self.db.query(sql_search)
size = self.get_song_nums('sname', author=singer)
print('数据库目前收录%d首' % size)
print '分别有:'
for l in data:
print(l[0])
print('是否开始下载?(y/n)')
choice = raw_input()
if choice == 'y':
base_dir = self.cm.get_config('dir')['path']
download.download_with_singer(data, base_dir, singer, size)
else:
print '已取消下载'
def searchBySname(self, song, singer=None):
"""
提供对外调用的接口, 接受歌曲名作为参数
:param song: 歌曲名
:param singer: 歌手名(选项)
:return:
"""
table = '%sresult' % self.cm.get_config('table')[0]['song']['prefix']
sql_search = ('SELECT author, durl FROM %s ' % table) + 'WHERE sname = \'%s\'' % song
data = self.db.query(sql_search)
size = self.get_song_nums('author', sname=song)
print('数据库目前收录%d首' % size)
print '分别有:'
for l in data:
print(l[0])
print('是否开始下载?(y/n)')
choice = raw_input()
if choice == 'y':
base_dir = self.cm.get_config('dir')['path']
download.download_with_sname(song, data, base_dir)
else:
print '已取消下载'
# coding: utf-8
import build.setup as setup
from api.baidu import Baidu
from task.dataprocessor import DataProcedure
from lib.songcrawler import Crawler
import build.clean as clean
from conf.cm import ConfigManager
cm = ConfigManager()
if __name__ == '__main__':
setup.setup_env()
clean.cleanup()
# targets = cm.get_config('target')['authors']
# for author in targets:
# tmp = author.decode('utf-8')
# print u'开始下载%s的歌曲' % tmp
# Crawler().get_url_by_singer(author)
# DataProcedure().run()
# Baidu().searchBySinger(tmp)
# 测试get_url_by_sname功能
Crawler().get_url_by_sname('当爱已成往事')
DataProcedure().run()
Baidu().searchBySname(u'当爱已成往事')
