def select_to_merge():
sql = """
select * from crawler_online.download_media where merged_order not in (0,-1) and download_status=1
and merged_status is null and download_path!='/data/dev_ant/';
"""
client = ConfigInit().get_conn()
return client.getAll(sql)
def select_not_download_over_file():
sql = """
SELECT absolute_path FROM crawler_online.download_media where download_status!=1;
"""
client = ConfigInit().get_conn()
all_tuple = client.getAll(sql)
return [url_dict['absolute_path'] for url_dict in all_tuple] if all_tuple else []
def select_original_url_downloaded_merged_media(urls_list):
sql = """
select original_url from crawler_online.download_media
where media_type='merged' and file_type='mp4' and download_status=1 and original_url in (%s)
;""" % ', '.join(map(lambda x: "'%s'" % x, urls_list))
client = ConfigInit().get_conn()
urls_tuple = client.getAll(sql)
return [url_dict['original_url'] for url_dict in urls_tuple] if urls_tuple else []
def select_tmp():
sql = """
select id,original_url,absolute_path,download_url,media_type,language,file_type
from crawler_online.download_media
"""
client = ConfigInit().get_conn()
all_tuple = client.getAll(sql)
return all_tuple
def select_original_url_downloaded_video_audio(urls_list):
sql = """
select original_url from crawler_online.download_media
where
(media_type='video' or media_type='audio' ) and download_status=1 and original_url in (%s)
group by original_url having count(original_url)>1;""" % ', '.join(map(lambda x: "'%s'" % x, urls_list))
client = ConfigInit().get_conn()
urls_tuple = client.getAll(sql)
return [url_dict['original_url'] for url_dict in urls_tuple] if urls_tuple else []
def select_to_merge():
sql = """
select * from download_media where merged_sign in
(select merged_sign from crawler_online.download_media where merged_status!=1 and merged_sign!='' and media_type!="%s"
group by merged_sign having count(merged_sign)>1)
order by merged_sign ,merged_order;
""" % consts.constant_manager.MERGED
client = ConfigInit().get_conn()
return client.getAll(sql)
def select_original_url_downloaded_subtitle(urls_list):
sql = """
select distinct original_url from download_media
where
media_type='subtitle' and download_status=1
and original_url in (%s);
""" % ', '.join(map(lambda x: "'%s'" % x, urls_list))
client = ConfigInit().get_conn()
urls_tuple = client.getAll(sql)
return [url_dict['original_url'] for url_dict in urls_tuple] if urls_tuple else []
def demo_del_file():
# 删除本地下载国产剧
urls = [url.replace('\n', '') for url in open('/data/my_ant/play_urls1')]
sql = """
SELECT absolute_path FROM crawler_online.download_media where download_status=1 and download_path='/data/dev_ant/' and
original_url in (%s);
""" % ', '.join(map(lambda x: "'%s'" % x, urls))
client = ConfigInit().get_conn()
all_tuple = client.getAll(sql)
dalu_files_local = [url_dict['absolute_path']
for url_dict in all_tuple] if all_tuple else []
for file in dalu_files_local:
try:
if os.path.exists(file):
del_file(file)
except:
traceback.print_exc()
pass