def connectUpdateCreateStatisticsDB(directory):
'''Connects to the statistics database used for updating the statistics of
all channels, updates it if necessary or creates it if it does not exist
:param path: The path of the database
:type path: string
:raises: :class:``sqlite3.Error: Unable to connect to database
:returns: Connection to the database
:rtype: sqlite3.Connection
'''
#Connect to database
dbCon = yta.connectDB(os.path.join(directory, "statistics.db"))
db = dbCon.cursor()
#Get database version
try:
r = db.execute("SELECT dbversion FROM setup ORDER BY id DESC LIMIT 1;")
version = r.fetchone()[0]
del r
except sqlite3.Error:
#No version field: new database
version = 0
if version < __statisticsdbversion__:
try:
#Perform initial setup
if version < 1:
#Set encoding
dbCon.execute("pragma encoding=UTF8")
#Create tables
cmd = """ CREATE TABLE setup (
id INTEGER PRIMARY KEY UNIQUE NOT NULL,
autoupdate BOOLEAN NOT NULL,
lastupdate INTEGER NOT NULL,
maxcount INTEGER NOT NULL,
dbversion INTEGER NOT NULL
); """
dbCon.execute(cmd)
cmd = """ CREATE TABLE channels (
id INTEGER PRIMARY KEY UNIQUE NOT NULL,
name STRING UNIQUE NOT NULL,
lastupdate INTEGER NOT NULL,
complete BOOLEAN NOT NULL
); """
dbCon.execute(cmd)
#Set db version
version = 1
db.execute(
"INSERT INTO setup(autoupdate,lastupdate,maxcount,dbversion) VALUES(?,?,?,?)",
(False, 0, 100000, version))
dbCon.commit()
except sqlite3.Error as e:
print("ERROR: Unable to upgrade database (\"{}\")".format(e))
dbCon.rollback()
yta.closeDB(dbCon)
sys.exit(1)
#Return connection to database
return dbCon
def writeDownloadedFile(dbPath, filePath, replace, videoID):
'''Write file containing Youtube IDs of all videos already archived
:param dbPath: Path of the archive database
:type dbPath: string
:param filePath: Path where the file containing all existing IDs should be written to
:type filePath: string
:param replace: Whether to replace the existing video in the archive database
:type replace: boolean
:param videoID: The new video id
:type videoID: string
'''
#Check if db exists
if not os.path.isfile(dbPath):
return
try:
with open(filePath, 'w+') as f:
#Connect to database
db = yta.connectDB(dbPath)
#Read IDs of all videos already in archive
r = db.execute("SELECT youtubeID FROM videos;")
for item in r.fetchall():
#Write IDs to file
if not (replace and videoID == item[0]):
f.write("youtube {}\n".format(item[0]))
yta.closeDB(db)
except sqlite3.Error:
return
def readInfoFromDB(dbPath):
'''Read playlist and language from database
:param dbPath: Path of the archive database
:type dbPath: string
:raises: :class:``sqlite3.Error: Unable to read from database
:returns: List with language code at index 0 and playlist at index 1
:rtype: list of string
'''
db = yta.connectDB(dbPath)
r = db.execute(
"SELECT language,playlist FROM channel ORDER BY id DESC LIMIT 1;")
item = r.fetchone()
yta.closeDB(db)
return [item[0], item[1]]
def createOrConnectDB(path):
'''Create database with the required tables
:param path: Path at which to store the new database
:type path: string
:raises: :class:``sqlite3.Error: Unable to create database
:returns: Connection to the newly created database
:rtype: sqlite3.Connection
'''
#Create database
dbCon = yta.connectDB(path)
#Create table
yta.createChannelTable(dbCon)
#Return database connection
return dbCon
def _updateSubdirStatistics(db, path, name, captions, amendCaptions, maxcount,
lastupdate, complete, apiKey):
'''Update the statistics for one subdir
:param db: Connection to the statistics database
:type db: sqlite3.Connection
:param path: The path of the subdir
:type path: string
:param name: The channel/subdir name
:type name: string
:param captions: Whether to check if captions were added since archiving the video (Default: False)
:type captions: boolean
:param amendCaptions: Whether to download the captions that were added since the video was archived
:type amendCaptions: boolean, optional
:param maxcount: The max number of videos allowed to update
:type maxcount: integer
:param lastupdate: Timestamp of the last update
:type lastupdate: integer
:param complete: Whether the last update was complete
:type complete: boolean
:param apiKey: The API-Key for the Youtube-API
:type apiKey: string
:raises: :class:``requests.exceptions.RequestException: Unable to connect to API endpoint
:returns: Number of update counts left
:rtype: integer
'''
#Print status
print("Updating \"{}\"".format(name))
#Connect to channel database
channelDB = yta.connectDB(os.path.join(path, "archive.db"))
#Perform update
updateTimestamp = int(time.time())
maxcount, complete = updateStatistics(channelDB, lastupdate, captions,
maxcount, apiKey, amendCaptions)
#Close channel db
yta.closeDB(channelDB)
#Write new info to database
db.execute(
"UPDATE channels SET lastupdate = ?, complete = ? WHERE name = ?;",
(updateTimestamp, complete, name))
return maxcount
def createNewTestDB(path):
'''Create a test db using the two ytacommon methods'''
#Connect
dbCon = ytacommon.connectDB(path)
#Create video table
ytacommon.createVideoTable(dbCon)
insert = "INSERT INTO videos(title,creator,date,timestamp,youtubeID,filename,checksum,language,width,height,resolution,statisticsupdated,filesize) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)"
dbCon.execute(
insert,
("Test", "Test", "2020-01-01", 1577836800, "test", "test.mp4",
"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"en", 1920, 1080, "Full HD", 1577836800, 1000000))
#Create channel table
ytacommon.createChannelTable(dbCon)
insert = "INSERT INTO channel(name, url, playlist, language, videos, lastupdate, dbversion, maxresolution, totalsize) VALUES(?,?,?,?,?,?,?,?,?)"
dbCon.execute(
insert, ('', '', '', '', 0, 0, ytacommon.__dbversion__, "default", 0))
#Close
ytacommon.closeDB(dbCon)
def upgradeDB(request):
'''Prepare and upgrade database with versions given via the "internal_dbversion"
marker in the for of (oldversion, newversion), verify upgraded database version
number, yield connection to database, and close and delete it afterwards
'''
#Get database version
oldVersion = request.node.get_closest_marker("internal_dbversion").args[0]
newVersion = request.node.get_closest_marker("internal_dbversion").args[1]
#Prepare test database
dbPath = prepareAndUpgradeDatabase(oldVersion)
#Connect to database
_dbCon = ytacommon.connectDB(dbPath)
#Verify version
r = _dbCon.execute(
"SELECT dbversion FROM channel ORDER BY id DESC LIMIT 1;")
assert r.fetchone()[0] >= newVersion
yield _dbCon
ytacommon.closeDB(_dbCon)
utils.deleteIfExists(dbPath)
def addMetadata(args):
'''Add additional metadata to archive database
:param args: The command line arguments given by the user
:type args: list
'''
#Get database path
parser = argparse.ArgumentParser(
prog="ytameta",
description="Add additional metadata to existing archive databases")
parser.add_argument(
"DIR", help="The directory containing the archive database to work on")
args = parser.parse_args(args)
path = os.path.normpath(os.path.abspath(args.DIR))
dbPath = os.path.join(path, "archive.db")
if not os.path.isdir(path) or not os.path.isfile(dbPath):
parser.error("DIR must be a directory containing an archive database")
#Check if database needs upgrade
yta.upgradeDatabase(dbPath)
#Connect to database
dbCon = yta.connectDB(dbPath)
db = dbCon.cursor()
#Save thumbnails to database
r = db.execute("SELECT youtubeID FROM videos;")
for item in r.fetchall():
#Get video filepath
youtubeID = item[0]
try:
[timestamp, duration, tags, _, _, _, _, _] = getMetadata(youtubeID)
db.execute(
"UPDATE videos SET timestamp = ?, duration = ?, tags = ? WHERE youtubeID = ?",
(timestamp, duration, tags, youtubeID))
except yta.NoAPIKeyError:
break
except requests.exceptions.RequestException:
print("ERROR: Unable to load metadata for {}".format(youtubeID))
continue
#Close database
yta.closeDB(dbCon)
def backupDB(dbPath, backupDir):
'''Create a backup copy of the database in the specified directory
:param dbPath: Path of the database to back up
:type dbPath: string
:param backupDir: Path of the directory in which to store backup
:type backupDir: string
:raises: :class:``sqlite3.Error: Unable to backup database
:returns: True if backup successful, otherwise False
:rtype: boolean
'''
#Connect to database
con = yta.connectDB(dbPath)
#Check database integrity
if not checkDB(con):
print("ERROR: Database \'{}\' has integrity error".format(dbPath))
return False
#Create db backup
timestamp = int(time.time())
backupPath = os.path.join(backupDir, "{}.db".format(timestamp))
bck = sqlite3.connect(backupPath)
con.backup(bck)
bck.close()
#Zip backup
with ZipFile(backupPath + ".zip", 'w') as zipf:
zipf.write(backupPath, arcname="{}.db".format(timestamp), compress_type=ZIP_DEFLATED)
#Verify zip
with ZipFile(backupPath + ".zip", 'r') as zipf:
if zipf.testzip():
print("ERROR: Database \'{}\' backup zip error".format(dbPath))
return False
#Remove uncompressed backup
os.remove(backupPath)
return True
def findMissing(args):
'''Find discrepancies between files and database
:param argv: The command line arguments given by the user
:type argv: list
'''
parser = argparse.ArgumentParser(prog="ytamissing", description="Find discrepancies between files and archive database in directory")
parser.add_argument("DIR", help="The directory to work on")
args = parser.parse_args(args)
path = os.path.normpath(os.path.abspath(args.DIR))
if not os.path.isdir(path):
parser.error("DIR must be a directory")
#Read IDs from file
fFiles = []
fIDs = []
cmd = ["exiftool", "-api", "largefilesupport=1", "-m", "-Comment", path]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in p.stdout.readlines():
line = line.decode("utf-8").strip()
if line.startswith("Comment"):
vid = line.split(':', 2)[2].strip()
fIDs.append(vid)
fFiles[-1]["id"] = vid
if line.startswith("=="):
fFiles.append({"name" : os.path.basename(line.split(' ', 1)[1].strip())})
if not fIDs:
print("No videos found in directory")
return
#Read IDs from database
dbPath = os.path.join(path, "archive.db")
aFiles = []
try:
#Check if database needs upgrade
yta.upgradeDatabase(dbPath)
db = yta.connectDB(dbPath)
r = db.execute("SELECT youtubeID,title FROM videos;")
for item in r.fetchall():
#Write ids to list
aFiles.append({"name" : item[1], "id" : item[0]})
except sqlite3.Error as e:
print(e)
return
if not aFiles:
print("No videos found in archive db")
return
#Compare IDs
found = False
for aFile in aFiles:
try:
fIDs.remove(aFile["id"])
except ValueError:
found = True
print("Video file \"{}\" missing (ID: {})".format(aFile["name"], aFile["id"]))
for fID in fIDs:
found = True
fFile = [f for f in fFiles if f["id"] == fID][0]
print("Video \"{}\" not in database (ID: {})".format(fFile["name"], fFile["id"]))
if not found:
print("No discrepancies between files and database")
#Close db
yta.closeDB(db)
def archiveAll(args):
'''Call archive script for all subdirs
:param args: The command line arguments given by the user
:type args: list
'''
#Set all to false for subsequent calls
args.all = False
#Set statistics to false for subsequent calls
updateStatistics = args.statistics
updateCaptions = args.captions
amendCaptions = args.amendcaptions
args.statistics = False
args.captions = False
args.amendcaptions = False
t1 = time.time()
#Get path
path = os.path.normpath(os.path.abspath(args.DIR))
#Check for progress file in directory
progressPath = os.path.join(path, "progress.json")
try:
with open(progressPath, 'r') as f:
progress = json.load(f)
if (t1 - progress["abortTime"]) > 3600:
progress = {"elapsed": 0}
os.remove(progressPath)
except OSError:
progress = {"elapsed": 0}
logFile = os.path.join(path, "log")
if "subdirs" in progress:
subdirs = progress["subdirs"]
counter = progress["counter"] - 1
channels = progress["channels"]
#Print message
if channels > 1:
print("CONTINUING ARCHIVING ALL {} CHANNELS IN \'{}\'\n".format(
channels, path))
else:
#Get subdirs in path
subdirs = [
os.path.join(path, name)
for name in sorted(os.listdir(path), key=str.casefold)
if os.path.isdir(os.path.join(path, name))
]
subdirs = [
sub for sub in subdirs
if os.path.isfile(os.path.join(sub, "archive.db"))
]
random.shuffle(subdirs)
if not subdirs:
print("ERROR: No subdirs with archive databases at \'{}\'".format(
path))
return
#Prepare
channels = len(subdirs)
progress["channels"] = channels
counter = 0
with open(logFile, 'w') as f:
f.truncate()
#Print message
if channels > 1:
print("ARCHIVING ALL {} CHANNELS IN \'{}\'\n".format(
channels, path))
#Initiate error log
errorLog = ""
#Loop through all subdirs
try:
t2 = time.time()
leftover = subdirs.copy()
for subdir in subdirs:
counter += 1
name = os.path.basename(os.path.normpath(subdir))
args.DIR = subdir
args.LANG = None
args.VIDEO = None
print("\nARCHIVING \'{}\' ({}/{})".format(name, counter, channels))
archive(args, True)
#Read errors from log
error = ""
with open(os.path.join(subdir, "log"), 'r') as f:
lines = f.readlines()
for i in range(len(lines)):
if lines[i].startswith("ERROR"):
error += "\n" + lines[i - 1] + lines[i]
if error:
errorLog += '\n\n' + name + '\n' + error
leftover.remove(subdir)
except KeyboardInterrupt:
#Aborting, write progress file and log
t = time.time()
progress["counter"] = counter
progress["subdirs"] = leftover
progress["elapsed"] += t - t2
progress["abortTime"] = t
with open(progressPath, 'w') as f:
json.dump(progress, f)
if errorLog:
with open(logFile, 'a') as f:
f.writelines(errorLog)
#Rethrow exception
raise
#Progress file no longer relevant, removing it
try:
os.remove(progressPath)
except OSError:
pass
#Write error log
if not errorLog:
errorLog = "No errors\n"
with open(logFile, 'a') as f:
f.writelines(errorLog)
t3 = time.time()
#Check if statistics is set to autoupdate
autoUpdateStatistics = False
if not updateStatistics or updateCaptions:
try:
statsDB = yta.connectDB(os.path.join(path, "statistics.db"))
r = statsDB.execute(
"SELECT autoupdate FROM setup ORDER BY id DESC LIMIT 1;")
autoUpdateStatistics = bool(r.fetchone()[0])
del r
except sqlite3.Error:
pass
finally:
try:
yta.closeDB(statsDB)
except sqlite3.Error:
pass
#Update statistics
if updateStatistics or autoUpdateStatistics or updateCaptions or amendCaptions:
statTime = True
try:
ytameta.updateAllStatistics(path, autoUpdateStatistics,
updateCaptions, amendCaptions)
except yta.NoAPIKeyError:
print(
"ERROR: Unable to update video statistics as no API key is available"
)
except RequestException as e:
print(
"ERROR: Unable to update video statistics due to connection error: \"{}\""
.format(e))
else:
statTime = False
#Print time
t4 = time.time()
print("\nTotal runtime: {}\nArchive runtime: {}".format(
yta.intervalToStr(progress["elapsed"] + (t4 - t1)),
yta.intervalToStr(progress["elapsed"] + (t3 - t2))))
if statTime:
print("Statistic runtime: " + yta.intervalToStr(t4 - t3))
print("\nDONE!")
def archive(args, parsed=False):
'''Archive youtube videos or playlists
:param args: The command line arguments given by the user
:type args: list
'''
#Parse arguments
if not parsed:
parser = argparse.ArgumentParser(
prog="ytarchiver",
description="Download and archive Youtube videos or playlists")
parser.add_argument(
"-a",
"--all",
action="store_const",
dest="all",
const=True,
default=False,
help=
"Run archiver for all subdirectories with archive databases. In this mode, LANG and VIDEO will always be read from the databases"
)
parser.add_argument("-c",
"--check",
action="store_const",
dest="check",
const="-c",
default="",
help="Check each file after download")
group = parser.add_mutually_exclusive_group()
group.add_argument("-s",
"--statistics",
action="store_const",
dest="statistics",
const=True,
default=False,
help="Update the video statistics")
group.add_argument(
"-u",
"--captions",
action="store_const",
dest="captions",
const=True,
default=False,
help=
"List videos where captions were added since archiving (forces -s)"
)
group.add_argument(
"-x",
"--amendcaptions",
action="store_const",
dest="amendcaptions",
const=True,
default=False,
help=
"Download captions were they were added since archiving (forces -u and consequently -s)"
)
parser.add_argument(
"-r",
"--replace",
action="store_const",
dest="replace",
const="-r",
default="",
help="Replace an existing video (a video ID has to be provided)")
group = parser.add_mutually_exclusive_group()
group.add_argument("-8k",
"--8K",
action="store_const",
dest="quality",
const="8k",
help="Limit download resolution to 8K")
group.add_argument("-4k",
"--4K",
action="store_const",
dest="quality",
const="4k",
help="Limit download resolution to 4K (default)")
group.add_argument("-hd",
"--HD",
action="store_const",
dest="quality",
const="hd",
help="Limit download resolution to full HD")
parser.add_argument("-V",
"--version",
action="version",
version='%(prog)s {}'.format(yta.__version__))
parser.add_argument("DIR", help="The directory to work in")
parser.add_argument(
"LANG",
nargs='?',
help="The video language (read from the database if not given)")
parser.add_argument(
"-f",
"--file",
action="store",
dest="file",
help="Read IDs to archive from a batch file with one ID per line")
group.add_argument(
"--filter",
action="store",
dest="filter",
default=None,
help=
"Filter videos to download using Youtube-dl's match filter option")
parser.add_argument(
"VIDEO",
nargs='?',
help=
"The Youtube video or playlist ID (read from the database if not given)"
)
args = parser.parse_args(args)
if args.all and args.file:
parser.error("-a cannot be used in combination with batch file")
if args.all and args.replace:
parser.error("-a cannot be used in combination with replace")
#Check if API key provided
yta.getAPIKey(True)
#Archive all subdirectories
if args.all:
archiveAll(args)
return
#Validate path
path = os.path.normpath(os.path.abspath(args.DIR))
if not os.path.isdir(path):
parser.error("An existing directory must be specified")
#Check if database exists
dbPath = os.path.join(path, "archive.db")
if not os.path.isfile(dbPath):
#No database found, ask to create one
while True:
q = input("New archive. Populate with channel info? [Y|n] ")
if not q:
q = 'y'
a = q[0].lower()
if a in ['y', 'n']:
break
if a == 'y':
ytainfo.add(dbPath)
else:
ytainfo.createEmpty(dbPath)
t1 = time.time()
#Check if database needs upgrade
yta.upgradeDatabase(dbPath)
#Check if ID and language are specified
if not args.LANG or (not args.VIDEO and not args.file):
#Try reading playlist and language from database
try:
(args.LANG, args.VIDEO) = readInfoFromDB(dbPath)
except (sqlite3.Error, TypeError):
#Try reading playlist and language from files
try:
with open(os.path.join(path, "language"), 'r') as f:
args.LANG = f.readline().strip()
with open(os.path.join(path, "playlist"), 'r') as f:
args.VIDEO = f.readline().strip()
except (IndexError, OSError):
parser.error(
"LANG and VIDEO must be specified if no database exists.")
#Update lastupdate field
updateTimestamp = int(time.time())
db = yta.connectDB(dbPath)
db.execute("UPDATE channel SET lastupdate = ? WHERE id = 1",
(updateTimestamp, ))
#Replace existing video
if args.replace:
try:
youtubeID = db.execute(
"SELECT youtubeID FROM videos WHERE youtubeID = ?;",
(args.VIDEO, )).fetchone()[0]
assert youtubeID
except (sqlite3.Error, TypeError, AssertionError):
print(
"ERROR: Unable to replace video with ID \"{}\" as it is not in the archive database"
.format(args.VIDEO))
return
#Get format string
if args.quality:
q = args.quality
else:
q = db.execute(
"SELECT maxresolution FROM channel WHERE id=1;").fetchone()[0]
dlformat = yta.getFormatString(q)
#Prepare download
dlfilePath = os.path.join(path, "downloaded")
dbPath = os.path.join(path, "archive.db")
writeDownloadedFile(dbPath, dlfilePath, args.replace, args.VIDEO)
dlpath = os.path.join(path, "ID%(id)s&%(title)s.%(ext)s")
postHook = PostHook(args.LANG, db, args.check, args.replace)
#Set options
ytdlOpts = {
"call_home": False,
"quiet": False,
"format": dlformat,
"ignoreerrors": True,
"download_archive": dlfilePath,
"writesubtitles": True,
"subtitleslangs": [args.LANG],
"writedescription": True,
"writethumbnail": True,
"outtmpl": dlpath,
"cachedir": False,
"youtube_include_dash_manifest": True,
"retries": 10,
"fragment_retries": 25,
"skip_unavailable_fragments": False,
"continuedl": True,
"extractor_args": {
"youtube": {
"player_client": ["android"]
}
},
"throttledratelimit": 100000,
"allow_playlist_files": False,
"post_hooks": [postHook.finished]
}
ytdlOpts["postprocessors"] = [{
"key": "FFmpegVideoConvertor",
"preferedformat": "mp4"
}, {
"key": "FFmpegMetadata"
}, {
"key": "EmbedThumbnail",
"already_have_thumbnail": False
}]
if args.filter:
ytdlOpts["match_filter"] = matchFilterFunc(args.filter)
#Check if archiving one video/playlist or using a batch file
if args.file:
with open(args.file, 'r', encoding="utf-8") as f:
url = readBatchURLs(f)
else:
url = [args.VIDEO]
#Prepare log
logFile = os.path.join(path, "log")
#Download
with DoubleLogger(logFile):
with yt_dlp.YoutubeDL(ytdlOpts) as ytdl:
ytdl.download(url)
#Print status
print("Download complete, updating database...")
#Update video number and totalsize
try:
db.execute(
"UPDATE channel SET videos = (SELECT count(id) FROM videos), totalsize = (SELECT sum(filesize) FROM videos) WHERE id = 1;"
)
except sqlite3.Error:
pass
#Update statistics
if args.statistics or args.captions or args.amendcaptions:
print("Updating video statistics...")
try:
ytameta.updateStatistics(db,
updateTimestamp,
args.captions,
amendCaptions=args.amendcaptions)
except yta.NoAPIKeyError:
print(
"ERROR: Unable to update video statistics as no API key is available"
)
except RequestException as e:
print(
"ERROR: Unable to update video statistics due to connection error: \"{}\""
.format(e))
#Close database
yta.closeDB(db)
#Print time
t2 = time.time()
print("DONE! Duration: " + yta.intervalToStr(t2 - t1))
#Remove download archive file
try:
os.remove(dlfilePath)
except OSError:
pass
def fix(args, parsed=False):
'''Update artist in database and metadata
:param args: The command line arguments given by the user
:type args: list
'''
#Parse arguments
if not parsed:
parser = argparse.ArgumentParser(
prog="ytafix", description="Fix wrong artist information")
parser.add_argument(
"-a",
"--all",
action="store_const",
dest="all",
const=True,
default=False,
help=
"Run fixer for all subdirectories with archive databases. In this mode, the ARTIST will always be read from the database"
)
parser.add_argument("DIR", help="The directory to work in")
parser.add_argument(
"ARTIST",
nargs='?',
help="The correct artist name (read from the database if not given)"
)
args = parser.parse_args(args)
#Run fixer for all subdirectories
if args.all:
fixAll(args)
return
#Validate path
path = os.path.normpath(os.path.abspath(args.DIR))
dbPath = os.path.join(path, "archive.db")
if not os.path.isdir(path) or not os.path.isfile(dbPath):
parser.error("DIR must be a directory containing an archive database")
#Check if database needs upgrade
yta.upgradeDatabase(dbPath)
#Connect to database
db = yta.connectDB(dbPath)
#Get correct artist
if args.ARTIST:
artist = args.ARTIST
else:
try:
r = db.execute("SELECT name FROM channel LIMIT 1;")
(artist, ) = r.fetchone()
del r
if not artist:
raise sqlite3.Error
except sqlite3.Error:
parser.error(
"No correct artist specified and unable to read it from the database"
)
#Read filenames and checksums from database
files = []
try:
r = db.execute("SELECT creator,filename,youtubeID FROM videos;")
for f in r.fetchall():
files.append({"artist": f[0], "name": f[1], "id": f[2]})
del r
except sqlite3.Error as e:
print(e)
return
found = False
for f in files:
#Compare artist
if f["artist"] != artist:
found = True
filepath = os.path.join(path, f["name"])
try:
#Change meta data
artist, title = fixVideo(filepath, f["id"], artist)
#Calculate checksums
checksum = yta.calcSHA(filepath)
#Update database
db.execute(
"UPDATE videos SET checksum = ?, creator = ? , title = ? WHERE youtubeID = ?",
(checksum, artist, title, f["id"]))
except requests.exceptions.HTTPError:
print("ERROR: Unable to fix \"{}\"".format(f["name"]))
continue
print("File \"{}\" fixed".format(f["name"]))
if not found:
print("No files to fix")
#Close database
yta.closeDB(db)
def check(args, parsed=False):
'''Perform integrity checks on files
:param args: The command line arguments given by the user
:type args: list
'''
if not parsed:
parser = argparse.ArgumentParser(
prog="ytacheck", description="Verify integrity of archived files")
parser.add_argument("DIR", help="The directory to work in")
parser.add_argument(
"-a",
"--all",
action="store_const",
dest="all",
const=True,
default=False,
help="Run checker for all subdirectories with archive databases")
parser.add_argument(
"-c",
"--check",
action="store_const",
dest="check",
const=True,
default=False,
help="Perform additional integrity check using ffmpeg")
args = parser.parse_args(args)
#Run checker for all subdirectories
if args.all:
checkAll(args)
return []
#Validate path
path = os.path.normpath(os.path.abspath(args.DIR))
dbPath = os.path.join(path, "archive.db")
if not os.path.isdir(path) or not os.path.isfile(dbPath):
parser.error("DIR must be a directory containing an archive database")
#Read filenames and checksums from database
files = []
errors = []
try:
#Check if database needs upgrade
yta.upgradeDatabase(dbPath)
db = yta.connectDB(dbPath)
r = db.execute("SELECT id,filename,checksum FROM videos;")
for f in r.fetchall():
files.append({"checksum": f[2], "name": f[1], "id": f[0]})
except sqlite3.Error as e:
sys.exit(
"ERROR: Unable to read from database (Error: \"{}\")".format(e))
for f in files:
filepath = os.path.join(path, f["name"])
#CHeck if file exits
if not os.path.isfile(filepath):
msg = "ERROR: File \"{}\" missing".format(f["name"])
print(msg)
errors.append(msg)
continue
#Check movie file
if args.check:
cmd = ["ffmpeg", "-v", "error", "-i", filepath, "-f", "null", "-"]
out, _ = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).communicate()
if out:
msg = "ERROR: File \"{}\" corrupt!".format(f["name"])
print(msg)
errors.append(msg)
else:
print("File \"{}\" check passed".format(f["name"]))
#Calculate checksums
checksum = yta.calcSHA(filepath)
if not f["checksum"]:
db.execute("UPDATE videos SET checksum = ? WHERE id = ?",
(checksum, f["id"]))
print("WARNING: File \"{}\" no checksum in database, adding {}".
format(f["name"], checksum))
else:
if f["checksum"] == checksum:
print("File \"{}\" checksums match".format(f["name"]))
else:
msg = "ERROR: Checksum mismatch for file \"{}\" (New checksum: {})".format(
f["name"], checksum)
print(msg)
errors.append(msg)
#Close database
yta.closeDB(db)
#Print status
if errors:
print("\nDONE, {} CORRUPTED FILE(S)".format(len(errors)))
else:
print("\nDONE, NO CORRUPTED FILE")
#Return errors
return errors