def rtmpdumpDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
if resolveHost:
url = domainToIPno(line['address'], verbose)
else:
url = line['address']
part1 = url.partition(' playpath=')
part2 = part1[2].partition(' swfVfy=1 swfUrl=')
if "kanal5play" in part2[2]:
if verbose:
printInfo1("This is from kanal5play\nAdding --live option to download command")
rtmpdumpOptions = "--realtime"
else:
rtmpdumpOptions = ""
cmd = (
"rtmpdump -o '%s.%s'"
" -r %s"
" -y %s"
" -W %s"
" %s"
% (line['name'].rstrip(), line['suffix'],
part1[0],
part2[0],
part2[2],
rtmpdumpOptions)
)
if verbose:
printInfo1("rtmpdump command: %s" % cmd)
return cmd
def parseXML(xmlRoot, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
vidBitRate = 0
vidWidth = 0
currentQuality = 0
lastQuality = 0
downloads = []
if name == "null":
trys = 0
printInfo2("Getting page title to use as file name...")
while True:
trys += 1
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
try:
html = urllib2.urlopen(url)
except urllib2.HTTPError, e:
onError(35, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(36, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
except:
def checkFirstSvtPage(firstPage, verbose):
haltOnError = True
if verbose:
printInfo2("Parsing page...")
soup = BeautifulSoup(firstPage)
items = soup.findAll(attrs={'data-json-href' : True})
try:
firstLink = items[0]['data-json-href']
except:
xmlCode = "Error"
return xmlCode
if verbose:
printInfo1("Found first link:")
print firstLink
else:
sys.stdout.write(".")
sys.stdout.flush()
linkOK, linkError = checkLink(firstLink, haltOnError, verbose)
if not linkOK:
onError(55,linkError)
else:
if verbose:
printInfo1("Link OK")
xmlCode = checkSecondSvtPage(firstLink, verbose)
return xmlCode
def retrievePiratePlayXML(apiBaseUrl, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
xmlCode = ""
trys = 0
if verbose:
printInfo1("Using %s as source for getting XML" % apiBaseUrl)
if verbose:
printInfo2("Parsing the response from pirateplay.se API...")
parseUrl = "%s/%s%s" % (apiBaseUrl, getStreamsXML, url)
printInfo2("\nGetting streams for %s ..." % parseUrl)
printScores()
while True:
trys += 1
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
break
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
try:
piratePlayXML = urllib2.urlopen(parseUrl)
except urllib2.HTTPError, e:
onError(35, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(36, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
def ffmpegDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
ffmpeg = getffmpegPath(verbose)
if resolveHost:
url = domainToIPno(line['address'], verbose)
else:
url = line['address']
if ffmpeg == ffmpegPath:
if verbose:
cmd = (
"%s -i %s"
" -loglevel verbose -acodec copy -vcodec copy -absf aac_adtstoasc -timeout 1000"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
cmd = (
"%s -i %s"
" -stats -loglevel fatal -acodec copy -vcodec copy -absf aac_adtstoasc -timeout 1000"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
elif ffmpeg == avconvPath:
if verbose:
cmd = (
"%s -i %s"
" -loglevel verbose -acodec copy -vcodec copy" # -absf aac_adtstoasc"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
cmd = (
"%s -i %s"
" -acodec copy -vcodec copy" # -absf aac_adtstoasc"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
onError(16, "You do not have either ffmpeg or avconv on the paths set in your config")
if verbose:
printInfo1("ffmpeg command: %s" % cmd)
return cmd
def addVideo(videos, videoLink, bitrate, verbose):
suffixHint = "mp4"
if verbose:
printInfo2("Adding video...\nLink: %s\nBit rate: %s\nSuffix hint: %s" % (videoLink, bitrate, suffixHint))
videos.append({"videoLink" : videoLink,
"bitrate" : bitrate,
"suffixHint": suffixHint})
return videos
def findReportedBitrates(url, verbose):
if verbose:
printInfo2("Finding reported bitrates...")
else:
sys.stdout.write(".")
sys.stdout.flush()
result = url.split(',', 1)[-1]
result = result.rstrip(',.mp4.csmil/manifest.f4m')
result = result.split(",")
return result
def compareDurations(expectedDuration, actualDuration, verbose):
if verbose:
printInfo2("Comparing durations...")
printInfo1("Expected duration: %s s" % expectedDuration)
printInfo1("Actual duration: %s s" % actualDuration)
if expectedDuration == 0:
durationsMatch = True
printWarning("Expected duration was 0\nSkipping checking...")
else:
if actualDuration + 2 > expectedDuration and actualDuration - 2 < expectedDuration:
durationsMatch = True
else:
durationsMatch = False
return durationsMatch
def composeXML(videos, subtitleLink, verbose):
xmlCode = []
if verbose:
printInfo2("Generating XML...")
printInfo1("Adding %s streams" % len(videos))
else:
sys.stdout.write(".")
sys.stdout.flush()
xmlCode.append("<streams>")
for index in range(0, len(videos)):
if verbose:
printInfo2("Adding video stream #%s..." % index)
print "Bitrate: %s" % videos[index]['reportedBitrate']
print "Video link: %s" % videos[index]['videoLink']
if subtitleLink:
print "Subtitle link: %s" % subtitleLink
print "Suffix hint: %s" % videos[index]['suffixHint']
else:
sys.stdout.write(".")
sys.stdout.flush()
if subtitleLink:
xmlCode.append(('<stream quality="%s kbps" subtitles="%s" suffix-hint="%s" required-player-version="0">') %
(videos[index]['reportedBitrate'],
subtitleLink,
videos[index]['suffixHint'])
)
else:
xmlCode.append(('<stream quality="%s kbps" suffix-hint="%s" required-player-version="0">') %
(videos[index]['reportedBitrate'],
videos[index]['suffixHint'])
)
xmlCode.append(videos[index]['videoLink'])
xmlCode.append('</stream>')
xmlCode.append('</streams>')
return xmlCode
def wgetDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
if resolveHost:
url = domainToIPno(line['subs'], verbose)
else:
url = line['subs']
cmd = (
"wget -O '%s.srt'"
" %s"
% (line['name'].rstrip(),
url)
)
if verbose:
printInfo1("wget command: %s" % cmd)
return cmd
def getDownloadCommands(line, verbose):
subCmd = ""
if verbose:
printInfo2("Composing download commands for line...")
print line
if line['address'].startswith("http"):
if verbose:
printInfo1("This should be downloaded with ffmpeg")
videoCmd = ffmpegDownloadCommand(line, verbose)
elif line['address'].startswith("rtmp"):
printInfo1("This should be downloaded with rtmpdump")
videoCmd = rtmpdumpDownloadCommand(line, verbose)
if line['subs']:
if verbose:
printInfo1("This should be downloaded with wget")
subCmd = wgetDownloadCommand(line, verbose)
return videoCmd, subCmd
def dlListPart(dlList, urlsOnly, setQuality, checkDuration, fileInfo, bestQuality, downloadAll, verbose):
url = ""
name = ""
downloads = []
dlList = open(dlList)
lines = dlList.readlines()
dlList.close()
if not urlsOnly:
for line in lines:
if verbose:
printInfo2("Parsing line: %s" % line)
if len(line) > 1 and not line.startswith("#"):
if line.startswith("http") and not url: # line is a url and url is not set
url = line
elif url and line.startswith("http"): # url is already set and line is a url
onError(7, "Two urls in a row. Second should be a file name")
else:
name = line
if name and not url:
onError(9, "First line was not a url")
elif url and name:
downloads = generateDownloads(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
url = ""
name = ""
if url:
onError(8, "Last url did not have a following name")
else:
name = "null"
for line in lines:
if len(line) > 1 and line.startswith("http"):
downloads = retrievePiratePlayXML(line, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
return downloads
def setPerms(myFile, verbose):
if verbose:
printInfo2("Setting ownership and permissions...")
printInfo2("Changing group to %s" % group)
os.chown(myFile, uid, gid)
printInfo2("Setting write permission for group")
os.chmod(myFile, mask)
# os.chmod(myFile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
print
def convertVideo(videoInFile, convertTo, reEncode, verbose):
keepOld = False
reDownload = False
fileAccepted = False
fileName, fileExtension = os.path.splitext(videoInFile)
fileExtension = fileExtension.lstrip(".")
if verbose:
printInfo1("File name: %s" % fileName)
printInfo1("File extension: %s" % fileExtension)
for extension in videoExtensions:
if extension == fileExtension.lower():
fileAccepted = True
if not fileAccepted:
onError(15, "%s is probably not a video file" % videoInFile)
if fileExtension.lower() == convertTo:
printWarning("Same out format chosen as existing\nWill not convert")
else:
if verbose:
printInfo2("Converting %s to %s format" % (videoInFile, convertTo))
printInfo2("Renaming %s to %s.bak" % (videoInFile, videoInFile))
os.rename(videoInFile, "%s.bak" % videoInFile)
if fileExtension.lower() == "flv" and convertTo == "mp4":
ffmpeg = getffmpegPath(verbose)
if reEncode:
if verbose:
printInfo2("Reencoding video...")
cmd = ("%s"
" -i %s"
" -qscale 0 -ar 22050 -vcodec %s"
" '%s.%s'"
% (ffmpeg,
"%s.bak" % videoInFile,
videoCodec,
fileName, convertTo)
)
else:
cmd = ("%s"
" -i %s -vcodec copy -acodec copy"
" '%s.%s'"
% (ffmpeg,
"%s.bak" % videoInFile,
fileName, convertTo)
)
while True:
printInfo1("Will convert")
if continueWithProcess(fileName, fileExtension, keepOld, reDownload,
"Will re convert\n", "Keeping old file\nNo converting\n", verbose):
process = runProcess(cmd, verbose)
def findVideos(domain, sourceCode, videos, verbose):
gotSeason = False
gotURL = False
printInfo1("\nSearching for links in code with the word '%s' in the url..." % videoText)
soup = BeautifulSoup(sourceCode)
for item in soup.fetch(['h2', 'a']):
if verbose:
printInfo1("\nParsing line: %s\n..." % item)
if item.contents:
if item.name == "h2" and seasonText in item.contents[0]:
season = HTMLParser().unescape(item.contents[0])
if verbose:
printInfo2("Found season text")
printInfo1("Season: %s" % season)
gotSeason = True
if item.name == "a" and videoText in item['href']:
episodeTitle = HTMLParser().unescape(item['title'])
url = item['href']
if verbose:
printInfo2("Found link to video")
printInfo1("Episode title: %s" % episodeTitle)
printInfo1("URL: %s" % url)
gotURL = True
if not gotSeason and not gotURL:
if verbose:
printInfo2("No valuable info in this item")
if gotURL:
if not gotSeason:
season = "None"
url = urljoin(domain, url)
if verbose:
printInfo1("Adding...")
printInfo1("URL: %s" % url)
printInfo1("Season: %s" % season)
printInfo1("Episode title: %s" % episodeTitle)
videos.append({'url': url,
'season': season,
'episodeTitle': episodeTitle})
gotSeason = False
gotURL = False
printInfo1("Found %s videos" % len(videos))
return videos
def getSubSize(subAddress, checkDuration, verbose):
subSize = "0"
trys = 0
gotAnswer = False
if checkDuration:
if verbose:
printInfo2("Probing for size of subtitle file...")
while True:
trys += 1
if trys > maxTrys:
onError(26, "Giving up after %s trys" % (trys - 1))
printWarning("Setting subtitle size to %s" % subSize)
gotAnswer = True
break
try:
sub = urllib2.urlopen(subAddress)
except:
printInfo2("Could not get subtitle size")
onError(41, "Undefined error")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Got an answer")
meta = sub.info()
if meta.getheaders:
subSize = meta.getheaders("Content-Length")[0]
else:
onError(21, "Could not get headers")
printWarning("Setting subsize to %s", subSize)
gotAnswer = True
break
if gotAnswer:
break
else:
printWarning("Subsize check disabled")
printWarning("Setting subsize to %s" % subSize)
printInfo1("Sub size: %s B" % subSize)
return subSize
def addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose):
streamDuration = getDuration(videoStream, checkDuration, verbose)
if verbose:
printInfo2("Adding download...")
if subtitles:
subSize = getSubSize(subtitles, checkDuration, verbose)
else:
printInfo2("No subtitles to download")
subSize = 0
if fileInfo:
name = "%s.%s" % (name, quality)
downloads.append({'address': videoStream.strip(),
'suffix': suffixHint,
'subs': subtitles,
'name': name,
'quality': quality,
'duration': streamDuration,
'subSize': subSize})
printInfo2("Added %s to download list" % quality)
return downloads
def svtplaydlXML(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
lookupLink = False
videos = []
if verbose:
printInfo2("Finding qualitys available...")
cmd = "svtplay-dl %s --list-quality" % url
output = runProcessReturnOutput(cmd, verbose)
output = output[0].split("\n")
for line in output:
if "hls" in line.lower():
vidBitRate = int(line.split(' ', 1)[-1].rsplit('\t', 1)[0])
if verbose:
printInfo1("Found HLS stream with %s kbps" % vidBitRate)
if not setQuality and vidBitRate > minVidBitRate and vidBitRate < maxVidBitRate:
lookupLink = True
elif setQuality and setQuality == vidBitRate:
lookupLink = True
if lookupLink:
if verbose:
printInfo2("Looking up link...")
cmd = "svtplay-dl %s -q %s -P HLS -g" % (url, vidBitRate)
output = runProcessReturnOutput(cmd, verbose)
output = output[0]
output = output.split("\n")
if svtplaydlVersion == 0:
videoLink = output[1]
elif svtplaydlVersion == 1:
videoLink = output[0]
myLink = urlparse(videoLink)
myLoc = myLink.netloc
if verbose:
printInfo1("Net location: %s" % myLoc)
if (myLoc.startswith('svtplay') or
myLoc.startswith('svtklipp') or
myLoc.startswith('svtarchive') or
myLoc.startswith('ed3.cdn.svt') or
myLoc.startswith("tv4play") or
myLoc.startswith('nordond') or
myLoc.startswith('mtgxse')):
videoLink = "%s%s" % (videoLink.split("m3u8", 1)[0], "m3u8")
if verbose:
printInfo1("Video link:")
print videoLink
videos = addVideo(videos, videoLink, vidBitRate, verbose)
lookupLink = False
if verbose:
printInfo2("Checking for subtitles...")
cmd = "svtplay-dl %s -S --force-subtitle -g" % url
output = runProcessReturnOutput(cmd, verbose)
subtitleLink = output[0]
if verbose:
if subtitleLink:
printInfo1("Subtitle link:")
print subtitleLink
else:
printInfo1("No subitles found")
xmlCode = composeXML(videos, subtitleLink, verbose)
xmlCode = '\n'.join(xmlCode)
return xmlCode
def checkVideoLink(videoLink, verbose):
index = 0
linkOK = False
checkQuality = True
haltOnError = False
videos = []
resolution = "0 x 0"
bitrate = 0
correctLinkSuffix = "/manifest.f4m"
oldPattern = "/z/"
newPattern = "/i/"
reportedBitrates = findReportedBitrates(videoLink, verbose)
if verbose:
printInfo2("Checking video link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
if videoLink.endswith(correctLinkSuffix):
LinkOK = True
if verbose:
printInfo1("Link ending OK")
printInfo2("Stripping off suffix %s" % correctLinkSuffix)
else:
sys.stdout.write(".")
sys.stdout.flush()
videoLink = videoLink.rstrip(correctLinkSuffix)
if verbose:
printInfo1("New link:")
print videoLink
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
onError(61, "Link did not end with %s" % correctLinkSuffix)
if verbose:
printInfo2("Looking for valid video links...")
else:
sys.stdout.write(".")
sys.stdout.flush()
while True:
testLink = videoLink.replace(oldPattern, newPattern)
testLink = "%s/index_%s_av.m3u8?null=0" % (testLink, index)
if verbose:
printInfo2("Checking video link #%s..." % index)
print "%s/index_%s_av.m3u8?null=0" % (testLink, index)
else:
sys.stdout.write(".")
sys.stdout.flush()
linkOK , linkError = checkLink(testLink, haltOnError, verbose)
if not linkOK:
if verbose:
printWarning("Did not get an answer for link #%s" % index)
else:
sys.stdout.write(".")
sys.stdout.flush()
break
else:
if checkQuality:
resolution, bitrate, codecLongName = findQuality(testLink, verbose)
if "mpeg-4" in codecLongName.lower() or "h.264" in codecLongName.lower():
suffixHint = "mp4"
else:
suffixHint = "unknown_suffix"
if verbose:
printInfo1("Resolution:")
print resolution
printInfo1("Bit rate: ")
print bitrate
printInfo1("Codec long name: ")
print codecLongName
printInfo1("Suffix hint: ")
print suffixHint
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
printWarning("Not checking quality")
if verbose:
printInfo2("Adding video...")
else:
sys.stdout.write(".")
sys.stdout.flush()
videos.append({"videoLink" : testLink,
"resolution" : resolution,
"bitrate" : bitrate,
"reportedBitrate" : reportedBitrates[index],
"codecLongName" : codecLongName,
"suffixHint": suffixHint})
index += 1
return videos
printInfo1("This video has the best quality yet")
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
lastQuality = currentQuality
else:
if downloadAll:
if verbose:
printInfo1("Adding this as we will download alla streams")
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
elif quality == "null":
if verbose:
printInfo1("Adding this stream as this is likely to be the only one")
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
else:
if not setQuality and vidBitRate > minVidBitRate and vidBitRate < maxVidBitRate:
if verbose:
printInfo2("Adding this stream as it matches our selection")
print "Minimum bitrate: %s kbps" % minVidBitRate
print "This streams bitrate: %s kbps" % vidBitRate
print "Maximum bitrate: %s kbps" % maxVidBitRate
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
elif not setQuality and vidWidth > minVidWidth and vidWidth < maxVidWidth:
if verbose:
printInfo2("Adding this stream as it matches our selection")
print "Minimum width: %s kbps" % minVidWidth
print "This streams width: %s kbps" % vidWidth
print "Maximum width: %s kbps" % maxVidWidth
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
elif setQuality:
if setQuality == vidBitRate or setQuality == vidWidth:
printInfo2("Adding this stream as it matches set quality")
downloads = addDownload(videoStream, checkDuration, subtitles, suffixHint, name, fileInfo, quality, verbose)
def finish(downloads, keepOld, reDownload, checkDuration, listOnly, convertTo, bashOutFile, verbose):
shouldBeDeleted = []
if not listOnly:
if downloads:
infoDownloaded = getVideos(downloads, keepOld, reDownload, checkDuration, verbose)
if convertTo:
if verbose:
printInfo2("Converting downloads...")
convertDownloads(downloads, convertTo, verbose)
else:
infoDownloaded = ""
onError(17, "Could not find any streams to download")
else:
infoDownloaded = ""
printInfo1("\nListing only")
printScores()
if bashOutFile:
if continueWithProcess(bashOutFile, bashSuffix, True, False,
"Will redownload\n", "Keeping old file. No download\n", verbose):
bashFile = open("%s.%s" % (bashOutFile, bashSuffix), "w")
bashFile.write("#!/bin/bash\n\n")
if downloads:
printInfo1("These files would have been downloaded:")
for line in downloads:
# print line
printInfo1("\nVideo name: %s.%s" % (line['name'].rstrip(), line['suffix']))
printInfo1("Video quality: %s" % line['quality'])
printInfo1("Video address: %s" % line['address'])
if line['subs']:
printInfo1("Subtitles name: %s.srt" % line['name'].rstrip())
printInfo1("Subtitles address: %s" % line['subs'])
else:
printInfo1("No subtitles found")
print "Duration: %s s" % line['duration']
if bashOutFile:
if line['address'].startswith("http"):
cmd = ffmpegDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
elif line['address'].startswith("rtmp"):
cmd = rtmpdumpDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
if line['subs']:
cmd = wgetDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
if bashOutFile:
bashFile.close()
st = os.stat("%s.sh" % bashOutFile)
os.chmod("%s.sh" % bashOutFile, st.st_mode | stat.S_IEXEC)
else:
printWarning("Could not find anything that would have been downloaded")
for line in infoDownloaded:
printInfo1("\nVideo: %s" % line['videoName'])
printScores()
if line['expectedDuration'] != "0.000":
printInfo1("Expected duration: %s" % (str(datetime.timedelta(seconds=round(float(line['expectedDuration']), 2)))))
if verbose:
printInfo1("Duration: %s ms" % line['duration'])
printInfo1("Duration: %s" % line['durationFormatted'])
if line['videoDlComment'] == dlCommentError:
printError(line['videoDlComment'])
shouldBeDeleted.append(line['videoName'])
else:
printInfo2(line['videoDlComment'])
if verbose:
printInfo1("Expected duration: %s s" % line['expectedDuration'])
printInfo1("Actual duration: %s s" % line['duration'])
if not compareDurations(float(line['expectedDuration']), float(line['duration']) / 1000, verbose):
printError("Durations does not match")
shouldBeDeleted.append(line['videoName'])
if verbose:
printInfo1("File size: %s b" % line['fileSize'])
printInfo1("File size: %s" % line['fileSizeMeasure'])
if verbose:
printInfo1("Overall bit rate: %s bps" % line['overallBitRate'])
printInfo1("Overall bit rate: %s" % line['overallBitRateMeasure'])
print
if verbose:
printInfo1("Video format: %s" % line['videoFormat'])
printInfo1("Video codec ID: %s" % line['videoCodecId'])
printInfo1("Video bit rate: %s bps" % line['videoBitRate'])
printInfo1("Video bit rate: %s" % line['videoBitRateMeasure'])
printInfo1("Width: %s px" % line['width'])
printInfo1("Height: %s px" % line['height'])
printInfo1("Frame rate: %s fps" % line['frameRate'])
if verbose:
printInfo1("Frame count: %s" % line['frameCount'])
print
printInfo1("Audio format: %s" % line['audioFormat'])
printInfo1("Audio codec ID: %s" % line['audioCodecId'])
printInfo1("Audio bit rate: %s bps" % line['audioBitRate'])
printInfo1("Audio bit rate: %s" % line['audioBitRateMeasure'])
if line['subLines'] != 'na':
printInfo1("\nSubtitles: %s" % line['subName'])
printScores()
if line['subDlComment'] == dlCommentError:
printError(line['subDlComment'])
shouldBeDeleted.append(line['subName'])
else:
printInfo2(line['subDlComment'])
if line['expectedSubSize'] != "0":
printInfo1("Expected file size: %s B" % line['expectedSubSize'])
printInfo1("File size: %s B" % line['subSize'])
printInfo1("Number of lines: %s" % line['subLines'])
else:
printWarning("\nNo subtitles downloaded")
if shouldBeDeleted:
print
printError("\nThese files should be deleted and re downloaded")
printScores()
for line in shouldBeDeleted:
printWarning(line)
print
def checkSubtitleLink(subtitleLink, verbose):
if verbose:
printInfo2("Checking subtitle link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
def findQuality(url, verbose):
width = 0
height = 0
bitrate = 0
codecLongName = ""
trys = 0
noFFmpeg = False
ffprobe = getffprobePath(verbose)
if verbose:
printInfo2("Looking up quality for stream with %s..." % ffprobe)
else:
sys.stdout.write(".")
sys.stdout.flush()
if ffprobe == ffprobePath:
if verbose:
printInfo1("Using %s to get video information" % ffprobePath)
cmd = "%s -loglevel error -show_format -show_streams %s -print_format xml" % (ffprobe, url)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after %s trys" % (trys - 1))
printWarning("Setting bitrate to %s" % bitrate)
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if not noFFmpeg:
try:
xmlRoot = ET.fromstring(output)
except:
onError(43, "Did not receive a valid XML")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML:")
print output
for xmlChild in xmlRoot:
if 'bit_rate' in xmlChild.attrib:
bitrate = xmlChild.attrib['bit_rate']
for innerChild in xmlChild:
if 'codec_long_name' in innerChild.attrib and not codecLongName:
codecLongName = innerChild.attrib['codec_long_name']
if 'width' in innerChild.attrib and not width:
width = innerChild.attrib['width']
if 'height' in innerChild.attrib and not height:
height = innerChild.attrib['height']
gotXML = True
else:
onError(40, "Can not detect duration")
printWarning("Setting bitrate to %s" % bitrate)
gotAnswer = True
gotXML = True
if gotAnswer and gotXML:
break
else:
if verbose:
printInfo1("Using %s to get video information" % avprobePath)
cmd = "%s -loglevel error -show_format -show_streams %s -of json" % (ffprobe, url)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after % trys" % (trys - 1))
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
else:
sys.stdout.write(".")
sys.stdout.flush()
output, error = process.communicate()
gotAnswer = True
break
if gotAnswer:
break
if gotAnswer:
jsonString = json.loads(output)
if verbose:
print "Full json: %s" % json.dumps(jsonString, sort_keys=True, indent=2)
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['width']):
width = json.dumps(jsonString['streams'][0]['width'])
if verbose:
print "Width: %s" % width
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['height']):
height = json.dumps(jsonString['streams'][0]['height'])
if verbose:
print "Height: %s" % height
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['format']['bit_rate']):
bitrate = json.dumps(jsonString['format']['bit_rate'])
if verbose:
print "Bitrate: %s" % bitrate
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['codec_long_name']):
codecLongName = json.dumps(jsonString['streams'][0]['codec_long_name'])
if verbose:
print "Codec long name: %s" % codecLongName
else:
sys.stdout.write(".")
sys.stdout.flush()
if width and height:
if verbose:
printInfo1("Found both width and height")
print "%s x %s" % (width, height)
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find width and height")
else:
printWarning("\nCould not find width and height")
if bitrate:
if verbose:
printInfo1("Found bitrate")
print bitrate
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find bitrate")
else:
printWarning("\nCould not find bitrate")
if codecLongName:
if verbose:
printInfo1("Found codec long name")
print codecLongName
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find codec long name")
else:
printWarning("\nCould not codec long name")
return ("%s x %s" % (width, height), bitrate, codecLongName)
def getVideos(downloads, keepOld, reDownload, checkDuration, verbose):
oldReDownload = reDownload
printInfo2("\nStarting downloads")
for line in downloads:
trys = 0
videoCmd, subCmd = getDownloadCommands(line, verbose)
while True:
trys += 1
if trys > maxTrys:
onError(29, "Tried to download video %s times\nSkipping..." % (trys - 1))
videoComment = dlCommentError
break
print
printInfo2("Downloading stream to %s.%s ..." % (line['name'].rstrip(), line['suffix']))
expectedDuration = round(float(line['duration']), 2)
printInfo1("Expected duration: %s" % str(datetime.timedelta(seconds=expectedDuration)))
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
printScores()
if continueWithProcess(line['name'].rstrip(), line['suffix'], keepOld, reDownload,
"Will redownload\n", "Keeping old file. No download\n", verbose):
if verbose:
printInfo2("Executing download command...")
print videoCmd
exitCode = runProcess(videoCmd, verbose)
if exitCode != 0:
printScores()
onError(30, "Failed. Process exited on %s" % exitCode)
printInfo2("Trying again...")
reDownload = True
else:
if checkDuration and float(line['duration']) > 0:
durationOK = checkDurations(line, verbose)
else:
if verbose:
printWarning("Not checking duration")
durationOK = True
if os.path.isfile("%s.%s" % (line['name'].rstrip(), line['suffix'])) and durationOK:
printScores()
printInfo1("Finished downloading video")
setPerms("%s.%s" % (line['name'].rstrip(), line['suffix']), verbose)
reDownload = oldReDownload
videoComment = dlCommentSuccess
break
else:
printScores()
if not durationOK:
onError(46, "Durations does not match")
printInfo2("Trying again")
else:
onError(31, "Failed. Video file does not exist")
printInfo2("Trying again...")
reDownload = True
else:
videoComment = dlCommentExist
break
if subCmd:
trys = 0
oldReDownload = reDownload
while True:
trys += 1
if trys > maxTrys:
onError(32, "Tried to download subtitles %s times\nSkipping..." % (trys - 1))
subComment = dlCommentError
break
print
printInfo2("Downloading subtitles %s.srt ..." % line['name'].rstrip())
if verbose:
printInfo1("from %s" % line['subs'])
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
printScores()
if continueWithProcess(line['name'].rstrip(), "srt", keepOld, reDownload,
"Will redownload\n", "Keeping old file. No download\n", verbose):
if verbose:
printInfo2("Downloading file...")
print line['subs']
print videoCmd
result = downloadFile(line['subs'], "%s.%s" % (line['name'].rstrip(), "srt"), verbose)
if not result:
printScores()
onError(33, "Failed to download subtitles")
printInfo2("Trying again...")
reDownload = True
else:
fileSizeOK = checkFileSize(line, verbose)
if os.path.isfile("%s.srt" % line['name'].rstrip()) and fileSizeOK:
printScores()
printInfo1("Finished downloading subtitles")
setPerms("%s.srt" % line['name'].rstrip(), verbose)
reDownload = oldReDownload
subComment = dlCommentSuccess
break
else:
printScores()
if not fileSizeOK:
onError(47, "Failed. File sizes does not match")
printInfo2("Trying again")
else:
onError(34, "Failed. Subtitle file does not exist")
printInfo2("Trying again")
reDownload = True
else:
subComment = dlCommentExist
break
else:
subComment = dlCommentNoSub
printInfo2("\nGetting file info...")
fileSize = getInfo(line, '--Inform="General;%FileSize%"', verbose)
fileSizeMeasure = getInfo(line, '--Inform="General;%FileSize/String%"', verbose)
duration = getInfo(line, '--Inform="General;%Duration%"', verbose)
durationFormatted = getInfo(line, '--Inform="General;%Duration/String3%"', verbose)
overallBitRate = getInfo(line, '--Inform="General;%OverallBitRate%"', verbose)
overallBitRateMeasure = getInfo(line, '--Inform="General;%OverallBitRate/String%"', verbose)
videoFormat = getInfo(line, '--Inform="Video;%Format%"', verbose)
videoCodecId = getInfo(line, '--Inform="Video;%CodecID%"', verbose)
videoBitRate = getInfo(line, '--Inform="Video;%BitRate%"', verbose)
videoBitRateMeasure = getInfo(line, '--Inform="Video;%BitRate/String%"', verbose)
width = getInfo(line, '--Inform="Video;%Width%"', verbose)
height = getInfo(line, '--Inform="Video;%Height%"', verbose)
frameRate = getInfo(line, '--Inform="Video;%FrameRate%"', verbose)
frameCount = getInfo(line, '--Inform="Video;%FrameCount%"', verbose)
audioFormat = getInfo(line, '--Inform="Audio;%Format%"', verbose)
audioCodecId = getInfo(line, '--Inform="Audio;%CodecID%"', verbose)
audioBitRate = getInfo(line, '--Inform="Audio;%BitRate%"', verbose)
audioBitRateMeasure = getInfo(line, '--Inform="Audio;%BitRate/String%"', verbose)
if line['subs']:
subSize = os.path.getsize("%s.srt" % line['name'].rstrip())
with open("%s.srt" % line['name'].rstrip()) as myfile:
subLines = sum(1 for line in myfile) # number of lines in file
myfile.close() # close file
else:
subSize = "na"
subLines = "na"
infoDownloaded.append({'videoName': "%s.%s" % (line['name'].rstrip(), line['suffix']),
'expectedDuration': line['duration'],
'videoDlComment': videoComment,
'fileSize': fileSize,
'fileSizeMeasure': fileSizeMeasure,
'duration': duration,
'durationFormatted': durationFormatted,
'overallBitRate': overallBitRate,
'overallBitRateMeasure': overallBitRateMeasure,
'videoFormat': videoFormat,
'videoCodecId': videoCodecId,
'videoBitRate': videoBitRate,
'videoBitRateMeasure': videoBitRateMeasure,
'width': width,
'height': height,
'frameRate': frameRate,
'frameCount': frameCount,
'audioFormat': audioFormat,
'audioCodecId': audioCodecId,
'audioBitRate': audioBitRate,
'audioBitRateMeasure': audioBitRateMeasure,
'subName': "%s.srt" % line['name'].rstrip(),
'expectedSubSize': line['subSize'],
'subDlComment': subComment,
'subSize': subSize,
'subLines': subLines})
printScores()
print
return infoDownloaded
def checkSecondSvtPage(url, verbose):
secondTag = ""
secondPage = getWebPage(url, verbose)
if verbose:
printInfo2("Parsing page...")
else:
sys.stdout.write(".")
sys.stdout.flush()
soup = BeautifulSoup(secondPage)
items = soup.findAll("embed", attrs={'attr' : True})
secondTag = items[0]['attr']
if secondTag:
if verbose:
printInfo1("Found second tag:")
print secondTag
printInfo2("Decoding...")
else:
sys.stdout.write(".")
sys.stdout.flush()
secondTag = urllib.unquote(secondTag.encode('utf8')).decode('utf8')
if verbose:
printInfo1("Decoded tag:")
print secondTag
secondTag = secondTag.split('=', 1)[-1]
else:
printError("Did not find second tag")
if verbose:
printInfo2("Converting to json...")
else:
sys.stdout.write(".")
sys.stdout.flush()
jsonString = json.loads(secondTag)
if verbose:
printInfo1("JSON string:")
print json.dumps(jsonString['video'], sort_keys=True, indent=2)
printInfo2("Extracting video link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
videoLink = jsonString['video']['videoReferences'][0]['url']
if verbose:
printInfo1("Found video link:")
print videoLink
else:
sys.stdout.write(".")
sys.stdout.flush()
videos = checkVideoLink(videoLink, verbose)
if verbose:
printInfo2("Extracting subtitle link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
if "url" in jsonString['video']['subtitleReferences'][0]:
subtitleLink = jsonString['video']['subtitleReferences'][0]['url']
checkSubtitleLink(subtitleLink, verbose)
if verbose:
printInfo1("Found subtitle link:")
print subtitleLink
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("No subtitles found")
else:
sys.stdout.write(".")
sys.stdout.flush()
subtitleLink = ""
if verbose:
printInfo1("Found videos:")
for video in videos:
print video
else:
sys.stdout.write(".")
sys.stdout.flush()
xmlCode = composeXML(videos, subtitleLink, verbose)
return xmlCode
def generateDownloads(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
xmlCode = ""
xmlRoot = ""
if name != name.replace("'", "").replace('"', '').replace(":", ""):
name = name.replace("'", "").replace('"', '').replace(":", "")
if verbose:
printWarning("Removed quotes , double quotes and colons in out file name")
if verbose:
printInfo2("Getting XML...")
s = 0
printInfo2("Trying these sources one by one:")
for xmlSource in xmlPriorityOrder:
s += 1
print "%s: %s" % (s, xmlSource)
for xmlSource in xmlPriorityOrder:
##### try internal XML generator #####
if xmlSource == "internal":
if verbose:
printInfo2("Getting XML from this programs python xml generator...")
xmlCode = internalXMLGenerator(url, verbose)
if xmlCode:
break
##### try local pirateplay's API #####
elif xmlSource == "localPiratePlay":
printInfo2("Getting XML from local pirateplay's API...")
xmlCode = retrievePiratePlayXML(apiBaseUrlLocal, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### try pirateplay's API #####
elif xmlSource == "piratePlay":
printInfo2("Getting XML from pirateplay.se's API...")
xmlCode = retrievePiratePlayXML(apiBaseUrlPiratePlay, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### try svtplay-dl #####
elif xmlSource == "svtPlayDl":
printInfo2("Getting XML from svtplay-dl...")
xmlCode = svtplaydlXML(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### giving up if no XML #####
if not xmlCode:
onError(69, "Could not find any way to get XML\nGiving up\nExiting!")
if verbose:
printInfo1("XML code:")
print xmlCode
try:
xmlRoot = ET.fromstring(xmlCode)
except:
onError(42, "Did not receive a valid XML")
#printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML")
downloads = parseXML(xmlRoot, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if len(downloads) == 0:
onError(67, "Did not find any suitable streams")# \nTrying another method...")
sys.exit()
return downloads
def getDuration(stream, checkDuration, verbose):
duration = "0.000"
gotAnswer = False
gotXML = False
noFFmpeg = False
trys = 0
ffprobe = getffprobePath(verbose)
if not ffprobe:
if verbose:
printWarning("Disabling checking of duration")
checkDuration = False
if checkDuration:
if verbose:
printScores()
printInfo2("Probing for duration of stream...")
if ffprobe == ffprobePath:
if verbose:
printInfo2("Using ffprope to determin duration...")
cmd = "%s -loglevel error -show_format -show_streams %s -print_format xml" % (ffprobe, stream)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after %s trys" % (trys - 1))
printWarning("Setting duration to %s" % duration)
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if not noFFmpeg:
try:
xmlRoot = ET.fromstring(output)
except:
onError(43, "Did not receive a valid XML")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML")
print output
for xmlChild in xmlRoot:
if 'duration' in xmlChild.attrib:
duration = xmlChild.attrib['duration']
if verbose:
printInfo1("Found duration in XML: %s" % duration)
gotXML = True
if not duration and verbose:
printWarning("Could not find duration in XML")
else:
onError(40, "Can not detect duration")
printWarning("Setting duration to %s" % duration)
gotAnswer = True
gotXML = True
if gotAnswer and gotXML:
break
if verbose:
printScores()
else:
printInfo2("Using avprobe to determine duration...")
cmd = "%s -loglevel error -show_format -show_streams %s -of json" % (ffprobe, stream)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after % trys" % (trys - 1))
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if gotAnswer:
break
if gotAnswer:
jsonString = json.loads(output)
if json.dumps(jsonString['format']['duration']):
duration = (json.dumps(jsonString['format']['duration']).strip('"'))
if verbose:
print "Duration: %s" % duration
else:
printWarning("Duration check disabled")
printWarning("Setting duration to %s" % duration)
printInfo1("Duration: %s s (%s)" % (duration.rstrip("0").rstrip("."),
str(datetime.timedelta(seconds=round(float(duration), 2))))
)
return duration
def convertDownloads(downloads, convertTo, verbose):
if verbose:
printInfo2("Converting the downloads to %s format" % convertTo)
