def un_compression(filename):
assertor.file_exist_assert(filename)
file_root = os.path.splitext(filename)[0]
judge_mkdir(file_root)
un_comp_file = Archive(filename)
un_comp_file.extractall(file_root)
def unarchive(list_files, dir_files, old_file):
for infile in list_files:
if infile[-3:] == "rar" or infile[-3:] == "zip":
archive = Archive(dir_files + infile)
archive.extractall(dir_files)
print("move archive: {}".format(infile))
shutil.move(dir_files + infile, old_file + infile)
def test_unrar(test_loc, test_file):
try:
rar_file = Archive(test_file)
rar_file.extractall(test_loc)
os.remove(test_loc + "/README.md")
return True
except Exception as ex:
partial_ex_end = str(ex).find("patool error")
print(str(ex)[:partial_ex_end])
return False
class UploadToDataSource(object):
def __init__(self, request, file_param, url_param, *args, **kwargs):
super(UploadToDataSource, self).__init__(*args, **kwargs)
self.request = request
self.file_param = file_param
self.url_param = url_param
if self.request.user.is_superuser and self.request.FILES.get(self.file_param, None):
self.zip_file_path = self.request.FILES[file_param].temporary_file_path()
elif self.url_param in self.request.POST.get(self.url_param, None):
download_req = requests.get(self.request.POST[self.url_param], stream=True, headers={'User-Agent': 'bernadvisorybot http://bernadvisory.org'})
with NamedTemporaryFile(delete=False) as f:
for chunk in download_req.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
self.zip_file_path = f.name
def unzip_file(self):
self.zip_ref = Archive(self.zip_file_path)
self.shapefile_dir = os.path.join(os.path.dirname(self.zip_file_path), os.path.basename(self.zip_file_path).split('.')[0] + "-unzipped")
self.zip_ref.extractall(self.shapefile_dir, auto_create_dir=True)
return True
def find_shapefile(self):
backup_file = None
# could fail if there's an infinitely recursive directory out there?
# :dice_roll_emoji:
for root, dirs, files in os.walk(self.shapefile_dir):
for _file in files:
if _file.endswith(".shp"):
self.shapefile_file = os.path.join(root, _file)
# this little construct gives preferential treatment
# to .shp files with keywords
# if re.search(r'precinct|votdst|politic', _file, flags=re.I):
return True
# and this is our backup
if self.shapefile_file:
return True
return False
def get_shapefile_data(self):
self.data_source = DataSource(self.shapefile_file)[0]
return self.data_source
def process(self):
if self.request.FILES['shapefile'].name.endswith('.zip'):
self.unzip_file()
if self.find_shapefile():
return self.get_shapefile_data()
# elif self.request.FILES['shapefile'].name.endswith('.shp'):
# self.shapefile_file = self.zip_file_path
# self.data_source = DataSource(self.shapefile_file)[0]
# return self.data_source
return "Sorry bro"
def decompress(filename, temp_dir):
"""
Attempts to decompress the input file to a temporary folder. If Patool fails
to unpack it, it is assumed that the file is not an archive.
:param filename: path to the input file
:param temp_dir: temporary directory for decompressed files
:return: list of files (these extracted if decompression was
performed or the input filename otherwise)
"""
# set the temporary folder name
base, ext = splitext(basename(filename))
base, ext2 = splitext(base)
if ext2 != '':
ext = ext2
tmp_dir = join(abspath(temp_dir), base)
# try to list files from the archive (do not re-decompress if files are
# already present)
if isdir(tmp_dir) and len(listdir(tmp_dir)) > 0:
logger.info("Listing files from '{}'...".format(filename))
try:
out = check_output(["patool", "list", filename])
files, bad = [], False
for line in out.split('\n'):
if line.startswith("patool: "):
break
fn = join(tmp_dir, line)
if not isfile(fn) and not ARCHIVE_EXCL(fn):
bad = True
break
if not ARCHIVE_EXCL(fn):
files.append(fn)
if not bad:
# if all required files are already decompressed, just return
# the list of file paths
return True, files
except CalledProcessError:
logger.debug("Not an archive, continuing...")
return False, [filename]
# now extract files
logger.info("Decompressing '{}' (if archive)...".format(filename))
archive = Archive(filename)
try:
archive.extractall(tmp_dir, auto_create_dir=True)
except (PatoolError, ValueError) as e:
if str(e).startswith("patool can not unpack"):
logger.debug("Not an archive, continuing...")
return False, [filename]
else:
logger.exception(e)
# retrieve the list of extracted files
return True, [join(tmp_dir, fn) for fn in listdir(tmp_dir) \
if not ARCHIVE_EXCL(fn)]
def extract_rar_content(install_dir):
# extract rar files
# unzip each zip
# calculate checksum of each RAR file
rar_file_title = install_dir + "ROMs.rar"
rar_file = Archive(rar_file_title)
if os.path.exists(install_dir + "ROMS.zip"):
os.remove(install_dir + "ROMS.zip")
rar_file.extractall(install_dir)
first_zip_title = install_dir + "ROMS.zip"
zip_sub_dir = install_dir
with zipfile.ZipFile(first_zip_title, "r") as zip_ref:
zip_ref.extractall(zip_sub_dir)
def test_unrar(test_loc, test_file):
try:
rar_file = Archive(test_file)
rar_file.extractall(test_loc)
os.remove(test_loc + "/README.md")
return True
except Exception as ex:
if "Permission denied" in str(ex):
print("Permission denied. Please retry with sudo AutoROM")
exit()
else:
partial_ex_end = str(ex).find("patool error")
print(str(ex)[:partial_ex_end])
return False
def unpack_all_archives(path, temp_path):
"""Unpacks all archives in folder and subfolders from path into temp_path
Keyword arguments:
path -- path to anime release
temp_path -- path to temp directory
"""
path = Path(path)
for i in path.iterdir():
if i.isdir():
unpack_all_archives(path, temp_path)
elif i.suffix.lower() in archive_extensions:
archive = Archive(str(i))
archive.extractall(temp_path)
def extract_file(out_folder,out_folder_as_filename=False):
'''
extract compressed file (zip,tar,tar.gz,rar)
arg:
1. filename = name of compressed file
2. url = dawnload url (default none)
3. dir = file directory
'''
archive_types= ['.7z' , '.ace' , '.alz' , '.a' , '.arc' , '.arj' , '.bz2' , '.cab' , '.Z',
'.cpio' , '.deb' , '.dms', '.gz' , '.lrz' ,'.lha', '.lzh' , '.lz' ,'.lzma' ,
'.lzo' , '.rpm' , '.rar' , '.rz' , '.tar' , '.xz' , '.zip', '.jar' , '.zoo']
for flname in os.listdir(out_folder):
if flname.endswith(*archive_types):
name= os.path.splitext(os.path.basename(flname))[0] # its spliting basename from file such('flname','zip')
if not os.path.isdir(name) : # scan the folder named name
try:
file_name=os.path.join(out_folder,flname)
archive=Archive(file_name)
if out_folder_as_filename == True: # extract file into new subfolder
new_folder=os.path.join(out_folder,name)
os.mkdirs(new_folder)
archive.extractall(new_folder)
else: # if arg is false extract to out_folder
archive.extractall(out_folder)
print('extraction {} to {} complete'.format(flname,new_folder))
except BadZipFile:
print('Unable to extract this file: {}, check the source it may be corrupt'.format(flname))
try: os.remove(file_name)
except OSError:
def extract_any_file(zip_file_path, destination):
print('Extracting {} into {}'.format(zip_file_path, destination))
zip_ref = Archive(zip_file_path)
zip_ref.extractall(destination)
print("Finished extraction!")
