def download(self, search, return_filename=False):
filename = f"{search}".replace(" ", "")
path = f'{app.config["SPLEETER_IN"]}{filename}'
ydl_opts = {
"format":
"bestaudio/best",
"extract_audio":
True,
"outtmpl":
f"{path}.%(ext)s",
"postprocessors": [{
"key": "FFmpegExtractAudio",
"preferredcodec": f"{app.config['YT_EXT']}",
"preferredquality": "999",
}],
"logger":
MyLogger(),
"progress_hooks": [my_hook],
"noplaylist":
True,
"max_downloads":
1,
"default_search":
"ytsearch",
}
ydl = YoutubeDL(ydl_opts)
ydl.download([search])
if return_filename:
return f"{filename}.{app.config['YT_EXT']}"
else:
with app.app_context():
return url_for("uploaded", filename=filename)
class ScramDLThread(threading.Thread):
def __init__(self, urls, addendum_params=None):
threading.Thread.__init__(self)
override_params = dict()
override_params['writeinfojson'] = False
override_params['test'] = False
override_params['format'] = 'mp4'
if addendum_params is not None:
override_params.update(addendum_params)
params = get_params(override_params)
self.scram_dl = YoutubeDL(params)
self.scram_dl.add_progress_hook(self.hook_progress)
self.scram_dl.add_default_info_extractors()
self.is_finished = False
self.urls = urls
def hook_progress(self, status):
if status['status'] == 'finished':
self.is_finished = True
else:
self.is_finished = False
def run(self):
self.scram_dl.download(self.urls)
def youtube_audio_download(urls, audio_format='wav'):
"""Downloads a list of youtube videos into audio files
Args:
urls (list(str)): List of youtube urls in string format or str of a single youtube video
audio_format (str, optional): Audio format of output file ('wav' or 'mp3') Defaults to 'wav'.
Returns:
file: returns an audio file for each url to /data/raw_audio/
"""
# Error in case the user inputs unsupported audio format
if audio_format not in ['mp3', 'wav']:
print("Audio Format not supported. Only 'wav' and 'mp3' supported.")
return 1
# If user inputs a str of 1 video then convert it to list format
if isinstance(urls, str):
urls = [urls]
ytdl_options = {
'format': 'bestaudio/best',
'extractaudio':True,
'audioformat':audio_format,
'outtmpl':'data/raw_audio/%(id)s.%(ext)s',
'noplaylist':True,
'nockeckcertificate':True,
'postprocessors':[{
'key': 'FFmpegExtractAudio',
'preferredcodec': audio_format,
'preferredquality': '192'
}]
}
ytdl = YoutubeDL(ytdl_options)
ytdl.download(urls)
def download_file(url: str, output_dir: str) -> Tuple[str, str]:
"""
Take a URL to a video, download the file and return the path to the audio
:param url: URL of video
"""
ops = {
'postprocessors': [{
'key': 'FFmpegExtractAudio'
}],
'outtmpl': os.path.join(output_dir, '%(id)s.%(ext)s'),
'keepvideo': True
}
ydl = YoutubeDL(ops)
try:
ydl.download([url])
except Exception as e:
log.error('Failed to download %s', url)
shutil.rmtree(output_dir)
raise
audio_file = None
video_file = None
video_exts = ['.mp4']
audio_exts = ['.m4a', '.mp3']
for f in os.listdir(output_dir):
if os.path.splitext(f)[1] in video_exts:
video_file = os.path.join(output_dir, f)
elif os.path.splitext(f)[1] in audio_exts:
audio_file = os.path.join(output_dir, f)
return audio_file, video_file
def download(self):
print('هلبيكسيد')
ytdl = YoutubeDL()
urlList = []
urlList.append(self.url)
ytdl.download(urlList)
print('بلخدمسيد')
def load_tracks(self, ydl: YoutubeDL, tracks: List[Track]):
for i, track in enumerate(tracks):
yield LoadStatus.STARTING, track, i, 0
# check if file already loaded
track_dir = track.set_output_path(self.output_dir, self.format,
self.tree, self.slugify)
os.makedirs(track_dir, exist_ok=True)
if os.path.exists(track.path):
yield LoadStatus.SKIPPED, track, i, 1
continue
# check if video exists
yield LoadStatus.SEARCHING, track, i, 0.1
track.fetch_video_url()
if not track.valid:
yield LoadStatus.FAILED, track, i, 1
continue
# load
yield LoadStatus.LOADING, track, i, 0.2
ydl.download([track.url])
# moving file
yield LoadStatus.MOVING, track, i, 0.8
src_path = os.path.join(self.output_dir, f'{track.video_id}.{self.format}')
shutil.move(src_path, track.path)
# restore meta data
yield LoadStatus.RESTORING_META, track, i, 0.9
track.restore_meta()
# fin
yield LoadStatus.FINISHED, track, i, 1
class YoutubeHandler():
def __init__(self) -> None:
self.youtube_api = build('youtube', 'v3', developerKey=YOUTUBE_API_KEY)
def search_video(self, search):
try:
search_response = self.youtube_api.search().list(
q=search,
part="id",
maxResults=1
).execute()
search_result = search_response.get("items", [])[0]
if search_result["id"]["kind"] == "youtube#video":
return search_result["id"]["videoId"]
except HttpError as e:
print("An HTTP error %d occurred:\n%s" %
(e.resp.status, e.content))
def download_audio(self, video_id, playlist_name):
self.audio_downloader = YoutubeDL(
{"format": "bestaudio/best",
'outtmpl': f'output/{playlist_name}/%(title)s.%(ext)s',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192'
}]})
self.audio_downloader.download(
[f"https://www.youtube.com/watch?v={video_id}"])
def index():
yt_url = request.args.get('url')
filename = YoutubeDL({
'forcefilename': True,
'quiet': True,
'simulate': True
})
#output = subprocess.check_output(filename.download([yt_url]))
#print(output)
ydl = YoutubeDL(ydl_opts)
ydl.download([yt_url])
#filename =
#script_dir = os.path.dirname(__file__)
#os.path.join(script_dir, filename)
#def generate_ytdl():
# x = 0
# while x <= 100:
# yield "data:" + str(x) + "\n\n"
# x = x + 10
# time.sleep(0.5)
# return
return "ok"
def test_info_json(self):
expected = list(EXPECTED_ANNOTATIONS) #Two annotations could have the same text.
ie = youtube_dl.extractor.YoutubeIE()
ydl = YoutubeDL(params)
ydl.add_info_extractor(ie)
ydl.download([TEST_ID])
self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
annoxml = None
with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
annoxml = xml.etree.ElementTree.parse(annof)
self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
root = annoxml.getroot()
self.assertEqual(root.tag, 'document')
annotationsTag = root.find('annotations')
self.assertEqual(annotationsTag.tag, 'annotations')
annotations = annotationsTag.findall('annotation')
#Not all the annotations have TEXT children and the annotations are returned unsorted.
for a in annotations:
self.assertEqual(a.tag, 'annotation')
if a.get('type') == 'text':
textTag = a.find('TEXT')
text = textTag.text
self.assertTrue(text in expected) #assertIn only added in python 2.7
#remove the first occurance, there could be more than one annotation with the same text
expected.remove(text)
#We should have seen (and removed) all the expected annotation texts.
self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
def test_info_json(self):
expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text.
ie = youtube_dl.extractor.YoutubeIE()
ydl = YoutubeDL(params)
ydl.add_info_extractor(ie)
ydl.download([TEST_ID])
self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
annoxml = None
with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
annoxml = xml.etree.ElementTree.parse(annof)
self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
root = annoxml.getroot()
self.assertEqual(root.tag, 'document')
annotationsTag = root.find('annotations')
self.assertEqual(annotationsTag.tag, 'annotations')
annotations = annotationsTag.findall('annotation')
# Not all the annotations have TEXT children and the annotations are returned unsorted.
for a in annotations:
self.assertEqual(a.tag, 'annotation')
if a.get('type') == 'text':
textTag = a.find('TEXT')
text = textTag.text
self.assertTrue(text in expected) # assertIn only added in python 2.7
# remove the first occurrence, there could be more than one annotation with the same text
expected.remove(text)
# We should have seen (and removed) all the expected annotation texts.
self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
def download(url, destination, logger):
opts = {
'format':
'm4a/bestaudio',
'outtmpl':
destination + '/%(title)s.%(ext)s',
'logger':
logger,
'progress_hooks': [dlProgress],
'postprocessors': [{
'key': 'MetadataFromTitle',
'titleformat': '%(artist)s - %(title)s'
}, {
'key': 'FFmpegMetadata'
}],
'ffmpeg_location':
resource_path('ffmpeg.exe')
}
ydl = YoutubeDL(opts)
ydl.add_default_info_extractors()
try:
ydl.download([url])
except:
app.enableButton("Download")
app.enableEntry("URL")
app.enableDirectoryEntry("destination")
app.queueFunction(writeOutput, sys.exc_info())
def download_mp3(yt_link):
if not re.match(r'((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?',
yt_link):
print("Invalid URL")
return False
ydl_options = {
'format': 'bestaudio/best',
'ext': 'mp3',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': f'{Path.cwd()}/yt_download/%(title)s.%(ext)s',
}
ydl = YoutubeDL(ydl_options)
try:
ydl.download([yt_link])
except:
pass
return True
def download(self, url=None, output=None):
opts = {
'outtmpl': os.path.join(output, '%(title)s-%(id)s.%(ext)s'),
'progress_hooks': [self.progress_hook],
}
y = YoutubeDL(params=opts)
y.download([url])
def process_url(self, source, global_opts):
"""Main method for processing urls
This method basically hands over the configuration to YoutubeDL and repeats
the step until every source in the configuration was read
"""
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url']
sourceDescription = source.get("description", "")
self._logsource(
"Processing source: '" + sourceDescription +
"' Url: '" + sourceUrl + "'", source)
# Merge local parameters with global ones
ydl.params = copy.copy(global_opts)
ydl.params.update(source)
prefix = ""
ydl.params['match_filter'] = (
None if 'match_filter' not in ydl.params or ydl.params['match_filter'] is None
else match_filter_func(ydl.params['match_filter']))
# Settings by commute tube over the standard settings, respect if the config sets them differently
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio/best" if 'format' not in self.config else self.config["format"]
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = self.download_archive
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
'%(uploader)s-%(title)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
elif not (ydl.params['outtmpl'].startswith(self.pathToDownloadFolder)):
self._logsource("Prefixing custom set outtmpl with '" + self.pathToDownloadFolder + "/" + prefix + "'", source)
ydl.params['outtmpl'] = self.pathToDownloadFolder + "/" + prefix + \
ydl.params['outtmpl']
if self.debug:
self._logsource(
"All downloads will be simulated since this is debug mode", source)
ydl.params['simulate'] = True
ydl.download([source['url']])
def download(self, url=None, output=None):
opts = {
'outtmpl':
os.path.join(
output, '%(title)s-%(id)s.%(ext)s'
),
'progress_hooks': [self.progress_hook],
}
y = YoutubeDL(params=opts)
y.download([url])
class ytmp3(object):
options = {
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': 'downloads/%(title)s.%(ext)s',
'ffmpeg_location': path.join(path.dirname(path.abspath(inspect.stack()[0][1])), "ffmpeg")
}
def __init__(self):
self.log = logging.getLogger("ytmp3")
ytmp3.options["logger"] = self.log
self.yts = ytsearch()
self.ytd = YoutubeDL(ytmp3.options)
def _load_file(self, filename):
pass
def search(self, query):
if not isinstance(query, str):
raise Exception("query must be a string type")
return self.yts.search(query)
def download(self, video_ids):
if not isinstance(next(iter(video_ids), None), str):
raise Exception("video ids must be a list of strings type")
self.ytd.download(video_ids)
def search_and_download(self, query, callback):
if not callable(callback):
raise Exception("callback must be a callable object")
results = self.search(query)
index = 0
if len(results) > 0:
index = callback(results)
if index >= 0 and index < len(results):
self.log.info(f"Downloading '{results[index].get('title')}'...")
self.download([results[index].get('id')])
self.log.info(f"Done!")
else:
raise Exception("callback returned invalid index")
def read_and_download(self, filename, delimeter = '\n'):
if not isinstance(filename, str) or not isinstance(delimeter, str):
raise Exception("filename and delimeter must be string types")
def index():
try:
url = request.args.get('url')
if url is None:
return response_with_cors(Response("Hawi wos is?")), 400
m = hashlib.md5()
m.update(url.encode('utf-8'))
directory = "tmp/" + m.hexdigest()
@after_this_request
def remove_file(response_to_return):
shutil.rmtree(directory)
return response_to_return
logger = CustomLogger("youtube-dl logger")
ydl = YoutubeDL({
"outtmpl":
directory + "/%(title)s.%(ext)s",
"max_filesize":
50000000,
"logger":
logger,
"postprocessors": [{
# use (any) ffmpeg postprocessor so it's run through ffmpeg at least once, so mime-type
# bits are set correctly and it can be shared via WhatsApp (otherwise it has application/octet-stream)
"key": "FFmpegMetadata"
}]
})
with ydl:
ydl.download([url])
files = [f for f in listdir(directory) if isfile(join(directory, f))]
messages = logger.getMessages()
if len(messages
) > 0 and "File is larger than max-filesize" in messages[-1]:
raise Exception("Max file-size exceeded.")
if len(files) == 0:
raise Exception("No files were downloaded?")
filename = files[0]
file_path = directory
return response_with_cors(
send_from_directory(file_path, filename, as_attachment=True))
except Exception as e:
app.logger.error(e)
return response_with_cors(Response(str(e))), 400
def get_cc(ids, langs):
opts = {
'skip_download': True,
'writesubtitles': True,
'subtitleslangs': langs.split(','),
'outtmpl': 'subs/%(id)s'
}
urls = ['https://youtube.com/watch?v={}'.format(id) for id in ids]
ydl = YoutubeDL(opts)
try:
ydl.download(urls)
except Exception as e:
print('Exception when downloading:', e)
def downloadSong(self, id):
options = {
# specify the path to download
'outtmpl': 'musicdownloads/%(id)s',
'postprocessors': [{
'key': 'FFmpegExtractAudio', # Tách lấy audio
'preferredcodec': 'mp3', # Format ưu tiên là mp3
'preferredquality': '192', # Chất lượng bitrate
}]
}
ydl = YoutubeDL(options)
# download() có thể truyền vào 1 str hoặc 1 list
ydl.download(["https://www.youtube.com/watch?v={}".format(id)])
def download(url, prefix=''):
try:
printf("=============Youtube Downloader: Initializing...========")
file_wav = ""
file_mp3 = ""
BASE_DOWNLOAD_DIR = os.path.normpath(os.getcwd() + '/static') + '/'
printf("**** Download dir: " + BASE_DOWNLOAD_DIR)
printf("=============Youtube Downloader: Setting options")
download_dir = BASE_DOWNLOAD_DIR
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
ydl_opts = {
'format': '140',
'quiet': True,
'progress_hooks': [
update_tqdm,
],
'outtmpl': download_dir + prefix + '%(title)s.%(ext)s',
'noplaylist': True
}
ydl_setup = YoutubeDL(ydl_opts)
info_dict = ydl_setup.extract_info(url, download=False)
title = info_dict.get('title', None)
title = title.replace('/', '_')
file_noext = download_dir + prefix + title
outfile = file_noext + '.m4a'
file_wav = file_noext + '.wav'
printf(">>>>>Start downloading " + url)
if not os.path.isfile(outfile):
x = threading.Thread(target=show_progress)
x.start()
ydl_setup.download([url])
printf('****Downloaded ' + outfile)
file_wav = convert(outfile, 'wav')
file_mp3 = convert(outfile, 'mp3')
x.join()
return file_wav, file_mp3
elif not os.path.isfile(file_wav):
printf('***** File already downloaded, converting to wav: ' +
outfile)
file_wav = convert(outfile, 'wav')
file_mp3 = convert(outfile, 'mp3')
return file_wav, file_mp3
else:
printf("**** Audio file(.wav) already exists: " + file_wav)
except Exception as e:
printf("****** Youtube Downloader went wrong")
print('====================', e, '======================')
def downloadSong(id):
options = {
# lấy tên file down về là id của video, lấy id làm tên file để tiện quản lí
'outtmpl':
'%(id)s',
'postprocessors': [{
'key': 'FFmpegExtractAudio', # Tách lấy audio
'preferredcodec': 'mp3', # Format ưu tiên là mp3
'preferredquality': '192', # Chất lượng bitrate
}]
}
ydl = YoutubeDL(options)
# download() có thể truyền vào 1 str hoặc 1 list
ydl.download(["https://www.youtube.com/watch?v={}".format(id)])
def download(link, files_present, avg):
'''gets the playlist and Downloads the videos that i dont have'''
#url = 'https://www.youtube.com/watch?v=MCs5OvhV9S4'
url = 'https://www.youtube.com/playlist?list=PLwyG5wA5gIzjhW36BxGBoQwUZHnPDFux3'
ydl=YoutubeDL()
ydl.add_default_info_extractors()
playlist = ydl.extract_info(url, download=False)
for video in playlist['entries']:
import ipdb; ipdb.set_trace()
if video['title'] in files_present:
print ("Video #{} {} is present and will be ignored").format(video['playlist_index'], video['title'])
else:
print ("currently downloading: {}").format(video['title'])
ydl.download(video['webpage_url'])
def download_video(video):
def test_hook(d):
if d['status'] == 'finished':
print("Done downloading. Starting conversion")
# end test_hook
options = {
#'verbose': True,
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'opus',
}],
'progress_hooks': [test_hook],
'outtmpl':
os.path.join(os.getcwd(), 'recordings', '%(id)s.%(ext)s')
}
print("Downloading video {0}".format(video))
downloader = YoutubeDL(options)
# Get all of the info
info_dict = downloader.extract_info(video, download=False)
video_url = info_dict.get("url", None)
video_id = info_dict.get("id", None)
video_title = info_dict.get('title', None)
video_duration = info_dict.get('duration', None)
# Download the video
downloader.download([video])
# by now, there should be a 'video_id.opus' file, transcode to ogg
assert (video_id + ".opus" in os.listdir('recordings'))
print("Transcoding {0}.opus to {0}.ogg".format(video_id))
file_noext = os.path.join(os.getcwd(), 'recordings', video_id)
os.system(
"""avconv -i {0}.opus -c:a libopus {0}.ogg -y""".format(file_noext))
retVal = metadata(video_title, video_id, video_url, file_noext + ".ogg",
video_duration)
retVal.thumbnail = info_dict.get("thumbnails")[0]['url']
os.remove(file_noext + ".opus")
print("""Video with ID {} is done. Pass to transcriber""".format(video_id))
return retVal
def ytdown(splitInput: str) -> None:
"Downloads youtube stream from share link"
fparser = argparse.ArgumentParser(prog="ytdown")
fparser.add_argument("-v","--verbose", help="Output everything", action="store_true")
fparser.add_argument("-q","--quiet", help="Do not print messages to stdout", action="store_true")
fparser.add_argument("-k","--keep", help="Keep video file after post processing", action="store_true")
fparser.add_argument("--skip", help="Skip video downloading", action="store_true")
fparser.add_argument("--geo_bypass", help="Bypass geographic restriction via faking X-Forwarded-For HTTP header", action="store_true")
fparser.add_argument("--prefer_ffmpeg", help="If False, use avconv instead of ffmpeg if both are available, otherwise prefer ffmpeg.", action="store_true")
fparser.add_argument("--proxy", help="URL of the proxy server to use")
fparser.add_argument("--thumbnails", help="Print a table of all thumbnails and exit", action="store_true")
fparser.add_argument("--source", help="Client-side IP address to bind to")
fparser.add_argument("-a","--archive", help="File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again")
fparser.add_argument("-u","--username", help="Username for authentication purposes")
fparser.add_argument("-p","--password", help="Password for authentication purposes")
fparser.add_argument("-f","--format", help="Video format code")
fparser.add_argument("-F","--formats", help="Print available formats", action="store_true")
fparser.add_argument("-s","--subtitle", help="Subtitle format code")
fparser.add_argument("-S","--subtitles", help="Print available subtitles", action="store_true")
fparser.add_argument("URL", help="URL to download")
try: fargs = fparser.parse_args(splitInput[1:])
except SystemExit: return
yt = YoutubeDL(dict({
"verbose":fargs.verbose,
"quiet":fargs.quiet,
"keepvideo":fargs.keep,
"skip_download":fargs.skip,
"geo_bypass":fargs.geo_bypass,
"prefer_ffmpeg":fargs.prefer_ffmpeg,
"list_thumbnails":fargs.thumbnails,
"source_address":fargs.source,
"download_archive":fargs.archive,
"username":fargs.username,
"password":fargs.password,
"format":fargs.format,
"listformats":fargs.formats,
"subtitlesformat":fargs.subtitle,
"listsubtitles":fargs.subtitles,
"format":fargs.format,
"format":fargs.format,
"format":fargs.format,
"format":fargs.format,
"format":fargs.format,
"format":fargs.format,
"format":fargs.format,
}))
yt.download([fargs.URL])
def start_youtube_dl(self):
# Start downloading the specified url
if self._output_path.get():
output_path = self._output_path.get()
else:
try:
output_path = os.path.dirname(os.path.abspath(__file__))
except NameError:
import sys
output_path = os.path.dirname(os.path.abspath(sys.argv[0]))
output_tmpl = output_path + '/%(title)s-%(id)s.%(ext)s'
options = {
'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio/best',
'merge_output_format': 'mp4',
'socket_timeout': '15',
'progress_hooks': [self._logger.log],
'ignoreerrors': True,
'outtmpl': output_tmpl,
}
if self._extract_audio.get():
options['format'] = 'bestaudio/best',
options['postprocessors'] = [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '3',
}]
dl = YoutubeDL(options)
status = dl.download([self._video_url.get()])
if status != 0:
mbox.showerror("youtube-dl error", "An error happened whilst processing your video(s)")
else:
mbox.showinfo("youtube-dl finished", "Your video(s) have been successfully processed")
def copy_youtubevideo_tos3(pk, url, bucket):
print('Starting copy_youtubevideo_tos3 task - pk: {0}; url: {1}'.format(
pk, url))
name = '{0}.mp3'.format(pk)
filename = '/tmp/videos/{0}'.format(name)
options = {'outtmpl': filename, 'format': 'bestaudio'}
ydl = YoutubeDL(options)
ydl_result = ydl.download([url])
print('Starting upload to Amazon S3...')
s3 = boto3.resource('s3')
s3_result = s3.Object(bucket, name).upload_file(filename)
print('Updating S3 URL...')
YoutubeVideo.objects.filter(pk=pk).update(s3_url=f's3://{bucket}/{name}')
print('End copy_youtubevideo_tos3 task - pk: {0}; url: {1}'.format(
pk, url))
return {
'locals': {
'pk': pk,
'url': url,
'filename': filename,
'bucket': bucket,
'name': name,
'ydl_result': ydl_result,
's3_result': s3_result
}
}
def run(self, release_task_pk: int, custom_options: Optional[dict]=None) -> None:
try:
with transaction.atomic():
self.download_task = DownloadTask.objects.get(pk=release_task_pk)
self.download_task.downloader = DownloadTask.YOUTUBE_DL
self.download_task.celery_id = self.request.id
self.download_task.state = DownloadTask.DOWNLOADING
self.download_task.started_at = timezone.now()
self.download_task.save()
options = copy.deepcopy(self.YOUTUBE_DL_OPTIONS)
options['outtmpl'] = options['outtmpl'].format(self.request.id.replace('-', ''))
options['progress_hooks'] = [self._progress_hook]
if custom_options:
custom_options = self._clean_options(custom_options)
options.update(custom_options)
downloader = YoutubeDL(options)
downloader.add_post_processor(self._get_postprocessor())
downloader.download([self.download_task.url])
except SystemExit:
try:
os.remove(self._last_status['tmpfilename'])
except AttributeError:
return
except:
log.exception('Exception while removing temporary file. id={0}, tempfilename={1}'.format(
release_task_pk, self._last_status['tmpfilename'])
)
raise
except (BaseDatabaseError, ObjectDoesNotExist, MultipleObjectsReturned):
# Hope we've caught everything
log.exception('Exception while updating DownloadTask. id={0}'.format(release_task_pk))
raise
except:
try:
with transaction.atomic():
self.download_task.state = DownloadTask.ERROR
self.download_task.finished_at = timezone.now()
self.download_task.save()
self.download_task.tracebacks.create(text=traceback.format_exc())
raise
except (BaseDatabaseError, ObjectDoesNotExist, MultipleObjectsReturned):
log.exception('Exception while changing DownloadTask.state to ERROR. id={0}'.format(release_task_pk))
raise
def on_task_output(self, task, config):
import youtube_dl.YoutubeDL
from youtube_dl.utils import ExtractorError
class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs):
self.to_stderr = self.to_screen
self.processed_info_dicts = []
super(YoutubeDL, self).__init__(*args, **kwargs)
def report_warning(self, message):
raise ExtractorError(message)
def process_info(self, info_dict):
self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict)
for entry in task.accepted:
if task.options.test:
log.info('Would download %s' % entry['title'])
else:
try:
outtmpl = entry.render(config['path']) + '/' + pathscrub(entry.render(config['template']) + '.%(ext)s', filename=True)
log.info("Output file: %s" % outtmpl)
except RenderError as e:
log.error('Error setting output file: %s' % e)
entry.fail('Error setting output file: %s' % e)
params = {'quiet': True, 'outtmpl': outtmpl}
if 'username' in config and 'password' in config:
params.update({'username': config['username'], 'password': config['password']})
elif 'username' in config or 'password' in config:
log.error('Both username and password is required')
if 'videopassword' in config:
params.update({'videopassword': config['videopassword']})
if 'title' in config:
params.update({'title': config['title']})
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
log.info('Downloading %s' % entry['title'])
try:
ydl.download([entry['url']])
except ExtractorError as e:
log.error('Youtube-DL was unable to download the video. Error message %s' % e.message)
entry.fail('Youtube-DL was unable to download the video. Error message %s' % e.message)
except Exception as e:
log.error('Youtube-DL failed. Error message %s' % e.message)
entry.fail('Youtube-DL failed. Error message %s' % e.message)
def _download_restricted(url, filename, age):
""" Returns true iff the file has been downloaded """
params = {
'age_limit': age,
'skip_download': True,
'writeinfojson': True,
"outtmpl": "%(id)s.%(ext)s",
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
json_filename = filename + '.info.json'
try_rm(json_filename)
ydl.download([url])
res = os.path.exists(json_filename)
try_rm(json_filename)
return res
def downloadYoutubePlaylist():
"""
Downloads the videos in the specified playlist in .mp3 format onto the local machine.
"""
ydl_opts = {
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
}
videos = ["TpigJNeY3RU", "mCxmPNGtuWQ"]
# ydl_opts = {}
ydl = YoutubeDL(ydl_opts)
ydl.download(videos)
def _download_restricted(url, filename, age):
""" Returns true iff the file has been downloaded """
params = {
'age_limit': age,
'skip_download': True,
'writeinfojson': True,
"outtmpl": "%(id)s.%(ext)s",
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
json_filename = filename + '.info.json'
try_rm(json_filename)
ydl.download([url])
res = os.path.exists(json_filename)
try_rm(json_filename)
return res
def test_info_json(self):
ie = youtube_dl.extractor.YoutubeIE()
ydl = YoutubeDL(params)
ydl.add_info_extractor(ie)
ydl.download([TEST_ID])
self.assertTrue(os.path.exists(INFO_JSON_FILE))
with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
jd = json.load(jsonf)
self.assertEqual(jd['upload_date'], u'20121002')
self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
self.assertEqual(jd['id'], TEST_ID)
self.assertEqual(jd['extractor'], 'youtube')
self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
self.assertTrue(os.path.exists(DESCRIPTION_FILE))
with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
descr = descf.read()
self.assertEqual(descr, EXPECTED_DESCRIPTION)
def test_info_json(self):
ie = youtube_dl.extractor.YoutubeIE()
ydl = YoutubeDL(params)
ydl.add_info_extractor(ie)
ydl.download([TEST_ID])
self.assertTrue(os.path.exists(INFO_JSON_FILE))
with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
jd = json.load(jsonf)
self.assertEqual(jd['upload_date'], u'20121002')
self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
self.assertEqual(jd['id'], TEST_ID)
self.assertEqual(jd['extractor'], 'youtube')
self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
self.assertTrue(os.path.exists(DESCRIPTION_FILE))
with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
descr = descf.read()
self.assertEqual(descr, EXPECTED_DESCRIPTION)
def start_download(video_url, pipe):
printLog('start downloading...')
#printLog('Child pipe count is: {0}'.format(sys.getrefcount(pipe)))
dir_name = str(uuid.uuid1()) # 下载临时目录
ydl_opts.update({
'outtmpl': 'tmp/' + dir_name + '/%(id)s.%(ext)s',
'progress_hooks': [progress_hook(pipe)]
})
youDLer = YoutubeDL(ydl_opts)
try:
youDLer.download([video_url]) # 阻塞函数，直到下载完毕后才会下一步
except Exception as e:
printLog('youtube-dl occure exception')
pipe.send(e) # 发送异常
time.sleep(0.1) # 等待接收
finally:
printLog('finally close child pipe.') # 不管是因为异常还是正常结束, 都关闭子进程端的管道
pipe.close()
def processUrl(self, source):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url'].decode()
sourceDescription = ""
if 'description' in source:
sourceDescription = source['description'].decode()
self.log.info(
"Processing source: '" + sourceDescription
+ "' Url: '" + sourceUrl + "'")
ydl.params = source
prefix = ""
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio"
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = "already_downloaded.txt"
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
u'%(uploader)s-%(title)s-%(id)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
if self.debug is True:
self.log.debug(
"All downloads will be simulated since this is debug mode")
ydl.params['simulate'] = True
ydl.download([source['url']])
def listen(self):
video_url = self.video_url.toPlainText()
print('video_url:', video_url)
save_path = self.save_path.text()
print('save_path:', save_path)
"""下载"""
params = {
"format": "bestvideo+bestaudio",
"outtmpl": save_path + os.sep + '%(title)s.%(ext)s',
"progress_hooks": [self.my_hook]
}
urls = []
for url in video_url.split('\n'):
urls.append(url)
yd = YoutubeDL(params)
yd.download(urls)
def processUrl(self, source):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url'].decode()
sourceDescription = ""
if 'description' in source:
sourceDescription = source['description'].decode()
self.log.info("Processing source: '" + sourceDescription + "' Url: '" +
sourceUrl + "'")
ydl.params = source
prefix = ""
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio"
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = "already_downloaded.txt"
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
u'%(uploader)s-%(title)s-%(id)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
if self.debug is True:
self.log.debug(
"All downloads will be simulated since this is debug mode")
ydl.params['simulate'] = True
ydl.download([source['url']])
def main(args, logger):
create_directory()
ytdl = YoutubeDL({
'quiet':
True,
'prefer_insecure':
True,
'outtmpl':
os.path.join(args.output_dir, '%(title)s.wav'),
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'wav'
}],
'keepvideo':
False
})
logger.info("Fetching playlist (this can take a while)")
playlist_name, playlist = scrapper.get_playlist_details(args.playlist_url)
saved_playlist = SavedPlaylist(playlist_name.lstrip().rstrip(),
args.mpd_playlist_dir)
removed_elements, remaining_elements = saved_playlist.compare_playlists(
playlist, args.output_dir, args.keep_removed)
if (removed_elements > 0):
logger.info(
'{} elements were removed from playlist'.format(removed_elements))
new_elements = len(remaining_elements)
logger.info("Found {} new elements, downloading...".format(new_elements))
for i in range(0, new_elements):
try:
ytdl.download([
'https://youtube.com/watch?v={}'.format(
remaining_elements[i]['upload_id'])
])
except Exception as e:
logger.error(e)
saved_playlist.remove_failed(remaining_elements[i:])
saved_playlist.save()
saved_playlist.save()
if new_elements > 0:
logger.finish("{} new elements added".format(new_elements))
def download_video(url,google_user_dict):
if google_user_dict["google_username"] != None and google_user_dict["google_passwd"]:
params = {"username":google_user_dict["google_username"],"password":google_user_dict["google_passwd"]}
else:
params = {}
downloader = YoutubeDL(params)
res = downloader.download([url])
filename = downloader.prepare_filename(res[1])
ret_code = res[0]
video_info = res[1]
return video_info
def download_audio_data(dataframe, save_path):
for idx, url in enumerate(dataframe["youtube_url"][1:]):
file_path = save_path + str(idx+1)
def my_hook(d):
if d['status'] == 'finished':
print('Done downloading {}, now converting ...'.format(str(idx+1) + ".mp3"))
if not os.path.isfile(save_path + ".mp3"):
ydl_opts = {
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': save_path + ".%(ext)s"} # ".%(ext)s" is necessary, otherwise downloaded file has no audio
ydl = YoutubeDL(ydl_opts)
ydl.download([url])
def download_mp3(url_list, download_folder='mp3'):
ydl = YoutubeDL()
ydl_opts = {
'quiet' : True,
'no_warnings' : True,
'verbose' : False,
'ignoreerrors':True,
#'verbosity': 'no_warnings',
'format': 'bestaudio/best',
'default_search': 'fixup_error',
'outtmpl': download_folder + os.sep + '%(title)s' + '.mp3',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
## actually, the download() function can accept a list.
## but then I lost error handling for indivual urls
## and the entire script would crash when one url was invalid
ydl.download(url_list)
# so now it's one by one
#for url in url_list:
# try:
# ydl.download(url_list)
# except youtube_dl.utils.DownloadError:
# print('Invalid URL:',url)
# #print(youtube_dl.utils.DownloadError)
# continue
#url_list = ['https://youtu.be/rBu0BRTx2x8']
#download_mp3(url_list)
def download_youtube(url,filepath,params=None):
tmp_filepath = compat_str(filepath)
print "download to",tmp_filepath
params = params or settings.youtube_params
params.update({"outtmpl":tmp_filepath,"daterange":DateRange(None,None)})
y = YoutubeDL(params)
#y = YoutubeDL({"format":"18/34/35/5/17","outtmpl":filepath})
#y.print_debug_header()
y.add_default_info_extractors()
y.add_post_processor(FFmpegExtractAudioPP(preferredcodec="m4a",preferredquality=5, nopostoverwrites=False))
value = y.download([url])
#cmd = 'youtube-dl {url} --extract-audio --audio-format wav -o {filepath}'.format(url=url,filepath=filepath)
#print cmd
#result = subprocess.call(cmd,shell=True)
#print result
return True
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie._WORKING:
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case and not test_case['file']:
print_skipping('No output file specified')
return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
params = self.parameters.copy()
params.update(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.fd.add_progress_hook(_hook)
test_cases = test_case.get('playlist', [test_case])
for tc in test_cases:
_try_rm(tc['file'])
_try_rm(tc['file'] + '.part')
_try_rm(tc['file'] + '.info.json')
try:
for retry in range(1, RETRIES + 1):
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
if retry == RETRIES: raise
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
raise
print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
else:
break
for tc in test_cases:
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
self.assertTrue(tc['file'] in finished_hook_called)
self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
if 'md5' in tc:
md5_for_file = _file_md5(tc['file'])
self.assertEqual(md5_for_file, tc['md5'])
with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected, compat_str) and expected.startswith('md5:'):
got = 'md5:' + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(expected, got,
u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n')
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
finally:
for tc in test_cases:
_try_rm(tc['file'])
_try_rm(tc['file'] + '.part')
_try_rm(tc['file'] + '.info.json')
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
info_dict = test_case.get('info_dict', {})
if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
print_skipping('The output file cannot be know, the "file" '
'key is missing or the info_dict is incomplete')
return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected, compat_str) and expected.startswith('md5:'):
got = 'md5:' + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(expected, got,
u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n')
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
# Check for mandatory fields that are automatically set by YoutubeDL
for key in ['webpage_url', 'extractor', 'extractor_key']:
self.assertTrue(info_dict.get(key), u'Missing field: %s' % key)
finally:
try_rm_tcs_files()
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case["name"])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get("add_ie", [])]
def print_skipping(reason):
print("Skipping %s: %s" % (test_case["name"], reason))
if not ie.working():
print_skipping("IE marked as not _WORKING")
return
if "playlist" not in test_case:
info_dict = test_case.get("info_dict", {})
if not test_case.get("file") and not (info_dict.get("id") and info_dict.get("ext")):
print_skipping(
'The output file cannot be know, the "file" ' "key is missing or the info_dict is incomplete"
)
return
if "skip" in test_case:
print_skipping(test_case["skip"])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u"test depends on %sIE, marked as not WORKING" % other_ie.ie_key())
return
params = get_params(test_case.get("params", {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status["status"] == "finished":
finished_hook_called.add(status["filename"])
ydl.fd.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get("file") or ydl.prepare_filename(tc.get("info_dict", {}))
test_cases = test_case.get("playlist", [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + ".part")
try_rm(os.path.splitext(tc_filename)[0] + ".info.json")
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case["url"]])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or (
err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503
):
raise
if try_num == RETRIES:
report_warning(u"Failed due to network errors, skipping...")
return
print("Retrying: {0} failed tries\n\n##########\n\n".format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get("params", {}).get("skip_download", False):
self.assertTrue(os.path.exists(tc_filename), msg="Missing file " + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + ".info.json"
self.assertTrue(os.path.exists(info_json_fn))
if "md5" in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc["md5"])
with io.open(info_json_fn, encoding="utf-8") as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get("info_dict", {}).items():
if isinstance(expected, compat_str) and expected.startswith("md5:"):
got = "md5:" + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(
expected, got, u"invalid value for field %s, expected %r, got %r" % (info_field, expected, got)
)
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict(
(key, value if not isinstance(value, compat_str) or len(value) < 250 else "md5:" + md5(value))
for key, value in info_dict.items()
if value and key in ("title", "description", "uploader", "upload_date", "uploader_id", "location")
)
if not all(key in tc.get("info_dict", {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(
u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u"\n"
)
# Check for the presence of mandatory fields
for key in ("id", "url", "title", "ext"):
self.assertTrue(key in info_dict.keys() and info_dict[key])
# Check for mandatory fields that are automatically set by YoutubeDL
for key in ["webpage_url", "extractor", "extractor_key"]:
self.assertTrue(info_dict.get(key), u"Missing field: %s" % key)
finally:
try_rm_tcs_files()
if len(news) > 0:
#: check news has any items
with open(file, 'w') as f:
f.write(news[0]['videoId'])
if len(news) > 0:
#: all new video file which will be download by youtube-dl
#: ydl_opts - should be check again, it now doesnot working :(
ydl = YoutubeDL()
os.getcwd()
os.chdir('/home/hadn/Downloads/adsense/football/')
ydl.add_default_info_extractors()
for value in news:
try:
ydl.download([value['link']])
for file_mp4 in os.listdir("/home/hadn/Downloads/adsense/football/"):
if file_mp4.endswith(".mp4"):
cmd = '/home/hadn/Downloads/adsense/python/bin/python /home/hadn/Downloads/adsense/donguyenha-upload-youtube.py --file "/home/hadn/Downloads/adsense/football/%s" --title "%s" --description "%s"' % (file_mp4, value['title'], value['title'])
cmd = shlex.split(cmd)
#: upload video to Youtube
up = Popen(cmd, stdout=PIPE)
temp = up.communicate()
videoId = temp[0].split()[-4]
add_video_to_playlist(videoId)
#: remove file which uploaded to youtube
os.remove(file_mp4)
#upload_blogger(value['title'], value['title'], "football, da banh", videoId)
except Exception, e:
print str(e)
pass
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
info_dict = test_case.get('info_dict', {})
if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
from sys import argv
from bs4 import BeautifulSoup
import requests
for i in argv[1:]:
data = requests.get(i)
soup = BeautifulSoup(data.content)
title = str(soup.title.text)
print title
os.getcwd()
os.chdir('/home/hadn/Downloads/adsense/football/')
ydl = YoutubeDL()
ydl.add_default_info_extractors()
try:
print i
ydl.download([i])
for file_mp4 in os.listdir("/home/hadn/Downloads/adsense/football/"):
if file_mp4.endswith(".mp4"):
cmd = '/home/hadn/Downloads/adsense/python/bin/python /home/hadn/Downloads/adsense/donguyenha-upload-youtube.py --file "/home/hadn/Downloads/adsense/football/%s" --title "%s" --description "%s"' % (file_mp4, title, title)
cmd = shlex.split(cmd)
print cmd
#: upload video to Youtube
up = Popen(cmd, stdout=PIPE)
temp = up.communicate()
videoId = temp[0].split()[-4]
#: remove file which uploaded to youtube
os.remove(file_mp4)
#upload_blogger(value['title'], value['title'], "football, da banh", videoId)
except Exception, e:
print str(e)
pass
