def createArchiveDir(self):
try:
os.mkdir(self.tmpdir)
except FileExistsError:
logger.out("Tried to make a temporary directory, but then realized I didn't need to...")
except Exception as e:
logger.out(e)
def __init__(self, args, setup):
# Create config object
self.conf = Config(setup.configdir+"/settings.json")
# args...
self.args = args
# Create manifest parser object
self.manifest = self.argsManifestFile()
self.manifest.parseManifest()
# Directories
self.setup = setup
self.tmpdir = self.setup.archivedir
self.extract = self.setup.archivedir+"extraction.csv"
self.migrate = self.setMigrationLocation()
# Process
logger.tear()
logger.out("Started packing mode...")
self.showStatusLast()
self.argsNoAsk()
self.argsManifestInfo()
self.calculateTotalSize()
self.prepairTempDir()
self.createArchiveDir()
self.createBackup()
def printManifest(self):
for key in self.order:
logger.out(key+":")
for i in self.data[key]:
logger.out("\t"+i)
return " "
def moveExtractedFiles(self):
# move extracted files from the temporary directory
# into the location specified in the extraction manifest
if os.path.isfile(self.extract):
# if the exctraction manifest exists, then extract according to its contents
self.extractUsingManifest()
else:
# otherwise abort
logger.out("[ ERROR ] No extraction manifest detected! Aborting.")
def printArchiveMembers(self):
members = []
archive = self.args.in_file
with tarfile.open(archive, "r|*") as tar:
members = tar.getnames()
for i in members:
logger.out(i)
logger.out("{0} files are ready for extraction.".format(len(members)))
def extractUsingManifest(self):
# open the extraction manifest and...
try:
with open(self.extract, mode='r', newline='\n') as csvFile:
csvData = csv.reader(csvFile, delimiter=',')
for row in csvData:
# for every item found there in, copy the file from the temp directory
# to the one contained in the current itteration of the for loop
# the extraction manifest data structure is: 0:child_file, 1:target_directory
self.moveFilesSafely(row[0], row[1])
except shutil.Error as e:
logger.out(e)
def createBackup(self):
for section in self.manifest.data:
# for every section of the manifest create a new tarball named after the manifest section
# if --section is passed, only backup section if it is specified
# check if archive file is in the way (Overwrite?)
tarname = "{tar}.tar.gz".format(tar=section)
tarpath = self.tmpdir+tarname
if section in self.listManifestSections():
if self.askOverwrite(self.migrate+tarname):
with tarfile.open(tarpath, "w:gz") as tar:
# for every file listed under the given section, add the file to the tarball
logger.out("Created new tar archive: {0}".format(tarpath))
self.createExtractionManifest()
for item in self.manifest.data[section]:
meta = self.createItemMetadata(item)
# Only add try to add the item to the tarball if it exists,
# otherwise inform the user of the non-existent entry
if os.path.isfile(item) or os.path.isdir(item):
self.addTarItem(tar, item, meta)
self.addMetadata(item, meta)
else:
logger.out("\"{i}\" was not added because it apperently dosen't exist...".format(i=item))
self.addExtractionManifest(tar)
self.migrateArchive(tarname)
else:
logger.out("Skiping: {0}".format(tarname))
logger.out("Finished!")
self.updateStatusLast()
def createKey(self):
logger.out("Creating key...")
try:
if not os.path.isfile(self.ui.keyFileLineEdit.text()):
#logger.out("dd if=/dev/urandom of={0} bs=1024 count=1".format(self.ui.keyFileLineEdit.text()))
key = subprocess.check_output("dd if=/dev/urandom of=\"{0}\" bs=1024 count=1".format(self.ui.keyFileLineEdit.text()), shell=True)
if self.ui.md5CheckBox.isChecked():
md5 = subprocess.check_output("md5sum \"{0}\"".format(self.ui.keyFileLineEdit.text()), shell=True)
print(md5.decode("utf-8").split(" ")[0])
with open(self.ui.keyFileLineEdit.text()+".md5", "w") as file:
file.write(md5.decode("utf-8").split(" ")[0])
except Exception as e:
logger.report()
raise CreationError(str(e))
def moveFilesSafely(self, item, target):
# copy file using shutil.move(), potentialy overwriting existing files, and creating directories as needed
try:
# Check if copying would result in overwriting files. if so, ask the user if it is ok to overwrite.
if (os.path.isdir(target+item) or os.path.isfile(target+item)):
#if the user answers no, return so as to avoid moving the files
if not self.askYesNo(msg="The file or directory \"{path}\" exists!/nWould you like to overwrite it?".format(path=target+item)):
logger.out("[ INFO ] Item skipped...")
return
if os.path.isdir(self.tmpdir+item):
copy_tree(self.tmpdir+item, target+item, verbose=1)
shutil.rmtree(self.tmpdir+item)
elif os.path.isfile(self.tmpdir+item):
shutil.copy2(self.tmpdir+item, target+item)
os.remove(self.tmpdir+item)
except FileNotFoundError as e:
logger.out("[ FATAL ] {}".format(e))
os._exit(1)
def askOverwrite(self, item):
# if the archive file already exists, ask to replace it... unless --noask is passed
if not self.args.noask:
if os.path.isfile(item):
print("{0} exists. Overwrite?".format(item))
while 1:
opt = input("[Y/N]")
if opt.lower()=="y":
return True
elif opt.lower()=="n":
return False
else:
print("Press \"Y\" or \"N\"!")
else:
return True
else:
logger.out("[ NOASK ]Question skipped in NoAsk mode...")
return True
def __init__(self, args, setup):
# Create config object
self.conf = Config(setup.configdir+"/settings.json")
# args...
self.args = args
# Directories
self.setup = setup
self.tmpdir = self.setup.archivedir
self.extract = self.setup.archivedir+"extraction.csv"
#print(self.args)
logger.tear()
logger.out(self.args, 0)
logger.out("Started unpacking mode...")
self.warnNoAsk()
self.argsCheckInFile()
self.printArchiveMembers()
if self.askYesNo("Continue?"):
self.extractArchive()
self.moveExtractedFiles()
def askYesNo(self, msg="Are you sure?", yes="yes", no="no"):
if not self.args.noask:
opt = input("{0} [{1}/{2}]:".format(msg, yes, no))
if opt == yes:
logger.out("User answered {0} to \"{1}\"".format(yes, msg), 0)
return 1
if opt == no:
logger.out("User answered {0} to \"{1}\"".format(no, msg), 0)
return 0
else:
print("You need to answer \"{0}\" or \"{1}\"".format(yes, no))
logger.out("User failed to give a proper answer to \"{0}\"".format(msg), 0)
return 0
else:
logger.out("[ WARN ] Question skipped in NOASK mode...")
return 1
def load(self):
try:
with open(self.default[0]["file"], 'r') as file:
self.data = json.load(file)
logger.out("Loaded settings...", 0)
except FileNotFoundError:
logger.out("Configuration not found. Will create a new one.", 0)
self.gen()
self.load()
except Exception as e:
logger.out("Failed to load configuration, could not continue...")
print(e)
sys.exit()
def calculateTotalSize(self):
logger.out("Calculating totals...")
# Total files to be compressed accross all archives
# Compressing X files and Y folders into Z archives
files = 0
folders = 0
archives = 0
for section in self.manifest.data:
if section in self.listManifestSections():
archives += 1
for i in self.manifest.data[section]:
if os.path.isdir(i):
folders += 1
else:
files += 1
if files+folders:
logger.out("Compressing {0} files and {1} folders into {2} archives...".format(files, folders, archives))
else:
logger.out("Nothing to backup! For help adding files and folders, type \"tp manifest --help\"")
def createVolume(self):
try:
logger.out("Creating Volume")
volume = self.ui.newVolLineEdit.text()
keyfile = self.ui.keyFileLineEdit.text()
sizeNumb = self.ui.sizeNumberSpinBox.value()
if os.path.isfile(self.ui.newVolLineEdit.text()):
reply = QMessageBox.question(self, "File Conflict", "Encryption will COMPLETELY overwrite this file!\nContinue?", QMessageBox.No, QMessageBox.Yes)
if reply == QMessageBox.No:
raise CreationError("User disallowed volume overwrite...")
#print("dd if=/dev/zero of=\"{0}\" bs=1 count=0 seek={1}{2}".format(self.ui.newVolLineEdit.text(), self.ui.sizeNumberDoubleSpinBox.value(), self.parseVolumeSize()))
vol = subprocess.check_output("dd if=/dev/zero of=\"{0}\" bs=1 count=0 seek={1}{2}".format(volume, sizeNumb, self.parseVolumeSize()),stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("cryptsetup luksFormat {0} {1} --batch-mode".format(volume, keyfile),stderr=subprocess.STDOUT , shell=True)
try:
vol = subprocess.check_output("cryptsetup luksOpen {0} luksLocker1 --key-file {1} --batch-mode".format(volume, keyfile),stderr=subprocess.STDOUT , shell=True)
except:
vol = subprocess.check_output("cryptsetup luksClose luksLocker1",stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("cryptsetup luksOpen {0} luksLocker1 --key-file {1} --batch-mode".format(volume, keyfile),stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("mkfs.ext4 /dev/mapper/luksLocker1",stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("mount /dev/mapper/luksLocker1 {0}".format(self.parent.conf.data[0]["mpoint"]),stderr=subprocess.STDOUT , shell=True)
#print("chown -R {0} {1}".format(self.parent.args.username, self.parent.conf.data[0]["mpoint"]))
vol = subprocess.check_output("chown -R {0} {1}".format(self.parent.args.username, self.parent.conf.data[0]["mpoint"]),stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("umount {0}".format(self.parent.conf.data[0]["mpoint"]),stderr=subprocess.STDOUT , shell=True)
vol = subprocess.check_output("cryptsetup luksClose luksLocker1",stderr=subprocess.STDOUT , shell=True)
self.parent.ui.volumeLineEdit.setText(volume)
except subprocess.CalledProcessError as e:
logger.out("ERROR: "+str(e))
logger.out(e.output.decode("utf-8"))
logger.report()
#print("THIS: ", e.output)
QMessageBox.warning(self, "Oops!", e.output.decode("utf-8"))
if self.parent.conf.data[0]['debug']:
QMessageBox.critical(self, "Warning", str(e))
raise CreationError(str(e.output.decode("utf-8")))
except Exception as e:
logger.report()
raise CreationError(str(e))
def migrateArchive(self, src):
old = self.tmpdir+src
new = self.migrate+src
logger.out("Moving archive {temp} to {dest}".format(temp=old, dest=new))
self.createFolderSafely(self.migrate)
shutil.move(old, new)
def info(self):
logger.out("#==== Mainfest Info ====#")
logger.out("| {0:21} |".format(self.man))
logger.out("#=======================#")
self.printManifest()
def showStatusLast(self):
logger.out("Last attempted backup was on {0}".format(self.conf.data["status"]["last"]))
def createMP(self):
try:
os.makedirs(self.parent.conf.data[0]["mpoint"])
except Exception as e:
logger.out("INFO: {0}".format(str(e)), wrt=0)
def argsCheckInFile(self):
if not self.args.in_file:
logger.out("What am I unpacking? Do \"tp unpack -if [archive]\"")
os._exit(1)
def warnNoAsk(self):
if self.args.noask:
logger.out("### WARNING NOASK FLAG HAS BEEN PASSED. SKIPPING ALL QUESTIONS ###")
def argsNoAsk(self):
if self.args.noask:
logger.out("[ NOASK ]No-ask mode! WILL OVERWRITE PRE-EXISTING ARCHIVES!")
def download(self):
# print(self.ytype)
if self.ytype == "PLIST":
#### DEBUGING ####
logger.out("Downloading Playlist", self.conf.data[0]["debug"])
logger.out(
tex="[" + str(self.conf.data[0]["limit"][0] - 1) + ":" + str(self.ajust_end_value()) + "]",
wrt=self.conf.data[0]["debug"],
)
# logger.out(str(len(self.playlist["items"]))+str(self.playlist["items"]), self.conf.data[0]["debug"])
##################
x = self.conf.data[0]["limit"][0]
for video in self.playlist["items"][self.conf.data[0]["limit"][0] - 1 : self.ajust_end_value()]:
best = video["pafy"].getbest(preftype="mp4")
filename = self.legalize(best.title) + "." + best.extension
self.title(
"{start} of {end} : {title}".format(start=x, end=len(self.playlist["items"]), title=best.title)
)
print("=================================================================")
print("Downloading video " + str(x) + " of " + str(len(self.playlist["items"])))
print(best.title)
print(
self.formatLocation()
+ self.legalize(self.playlist["title"])
+ "/"
+ self.numberedFilename(x, filename)
)
print("=================================================================")
try:
os.mkdir(self.formatLocation())
except:
pass
try:
os.mkdir(self.formatLocation() + self.legalize(self.playlist["title"]))
except:
pass
while 2 > 1:
try:
dl = Downloader(
url=best.url,
loc=self.formatLocation() + self.legalize(self.playlist["title"]),
name=self.numberedFilename(x, filename),
autoresume=True,
maxretry=self.conf.data[0]["maxRT"],
verbose=self.conf.data[0]["debug"],
)
dl.download(report)
break
except FileNotFoundError as e:
print(e)
self.conf._editlocation()
# urllib.urlretrieve(best.url, self.conf.data[0]["dlloc"]+self.legalize(self.playlist["title"])+"/"+filename, self.callback)
print()
x += 1
elif self.ytype == "VIDEO":
best = self.video.getbest(preftype="mp4")
filename = self.legalize(best.title + "-" + self.video.author) + "." + best.extension
try:
os.mkdir(self.formatLocation())
except:
pass
print("=================================================================")
print(best.title)
print("-----------------------------------------------------------------")
print(self.formatLocation() + filename)
print("=================================================================")
dl = Downloader(
url=best.url,
loc=self.formatLocation(),
name=filename,
autoresume=True,
maxretry=self.conf.data[0]["maxRT"],
verbose=self.conf.data[0]["debug"],
)
dl.download(report)
# urllib.urlretrieve(best.url, self.conf.data[0]["dlloc"]+filename, self.callback)
print()
elif self.ytype == "NONE":
pass
else:
raise TypeError('Expected: "VIDEO", "PLIST", or "NONE" for ytype value. Received:', self.ytype)
def addTarItem(self, tar, item, meta):
logger.out("adding: {0}".format(item))
tar.add(item, arcname=meta[0])