def convert_to_cbz(self, comic_file, comic_file_name):
### This function only converts comic's archive to cbz without deleting files or folders
### I do not know what will happen if there would be two folders inside archive. Maybe one day I'll find out.
cbz_comic_archive = ZipFile(self.comic_save_location +
comic_file_name + ".cbz",
mode="w",
compression=ZIP_STORED,
allowZip64=True)
# cbz_comic_archive = ZipFile(comic_save_location + comic_file_name + ".cbz", mode="w", compression=ZIP_STORED, allowZip64=True, compresslevel=None, strict_timestamps=True) # Commented out, because on python3.6 compresslevel and strict_timestamps are not supported
with TemporaryDirectory() as dir:
## Opens temporary directory named dir, that will be deleted when everything inside with statement is finished
comic = RarFile(comic_file,
mode="r") # Opening rar comic archive in read mode
comic.extractall(
path=dir) # Extracting every file to temporary directory
base = "" # To save folder if one exists.
for folder in walk(dir):
## Looping thought all folders/files
if folder[1] != []:
base = folder[1][0]
else:
for page in folder[2]:
# Adding every file in temporary dir to the archive
cbz_comic_archive.write(join(folder[0], page),
arcname=join(base + sep +
page))
class RarArchive(BaseArchive):
def __init__(self, file):
self._archive = RarFile(file)
def namelist(self):
return self._archive.namelist()
def filenames(self):
return self._archive.namelist()
def close(self):
return self._archive.close()
def is_encrypted(self):
return self._archive.needs_password()
def extraxt(self):
rarfile = rarfile.RarFile.self._archive
rarfile.extraxt(self._archive._rarfile)
def extractall(self, file):
return self._archive.extractall(file)
def list(self, *args, **kwargs):
self.printdir(*args, **kwargs)
def printdir(self, file=None):
"""Print a table of contents for the zip file."""
print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"),
file=file)
for file_ in self._archive._file_parser.infolist():
date = "%d-%02d-%02d %02d:%02d:%02d" % file_.date_time[:6]
print("%-46s %s %12d" % (file_.filename, date, file_.file_size),
file=file)
class RarFileExtractor(Extractor):
def __init__(self, archive_path: str):
self.archive_path = archive_path
self.rarfile = RarFile(self.archive_path)
self.namelist = {
i.filename
for i in self.rarfile.infolist() if not i.isdir()
}
def __exit__(self, *args):
self.rarfile.close()
def list_files(self) -> {}:
return self.namelist
def extract(self,
file_name: str,
dest_path: str,
write_hook: Callable[[bytes], bytes] = None):
with self.rarfile.open(file_name) as rf, open(dest_path, 'wb') as out:
while True:
chunk = rf.read(CHUNKSIZE)
if not chunk:
break
out.write(write_hook(chunk) if write_hook else chunk)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
subtitle.content = r.content
if subtitle.is_valid():
return
subtitle.content = None
raise ProviderError('Unidentified archive type')
subs_in_archive = archive.namelist()
# if Serbian lat and cyr versions are packed together, try to find right version
if len(subs_in_archive) > 1 and (subtitle.language == 'sr' or subtitle.language == 'sr-Cyrl'):
self.get_subtitle_from_bundled_archive(subtitle, subs_in_archive, archive)
else:
# use default method for everything else
subtitle.content = self.get_subtitle_from_archive(subtitle, archive)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
subtitle.content = r.content
if subtitle.is_valid():
return
subtitle.content = None
raise ProviderError('Unidentified archive type')
subs_in_archive = archive.namelist()
# if Serbian lat and cyr versions are packed together, try to find right version
if len(subs_in_archive) > 1 and (subtitle.language == 'sr'
or subtitle.language == 'sr-Cyrl'):
self.get_subtitle_from_bundled_archive(subtitle, subs_in_archive,
archive)
else:
# use default method for everything else
subtitle.content = self.get_subtitle_from_archive(
subtitle, archive)
def __append_tables_info(self, archive_file, inner_file_name,
inner_file_ext):
try:
inner_file = archive_file.read(inner_file_name)
binary = io.BytesIO(inner_file).read()
except (BadZipfile, BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError(f'{inner_file_ext}-архив не может быть прочитан')
if inner_file_ext == 'zip':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
zip_file = ZipFile(file_read, 'r')
for file_name in zip_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(zip_file, file_name, file_ext)
except BadZipfile:
raise TypeError('zip-архив не может быть прочитан')
elif inner_file_ext == 'rar':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
rar_file = RarFile(file_read)
for file_name in rar_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(rar_file, file_name, file_ext)
except (BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError('rar-архив не может быть прочитан')
parser = self.__choose_parser(inner_file_ext)(binary=binary, html=None)
tables_info = parser.get_tables_info()
for ti in tables_info:
self.tables_info.append(ti)
def download_demo(demo, event_folder):
print('Downloading %s' % demo['url'])
with requests.get(demo['url'],
allow_redirects=not DEBUG,
headers={'User-Agent': 'joder'},
stream=True) as r:
r.raise_for_status()
if DEBUG:
print(r.headers['Location'])
else:
local_filename = os.path.join(event_folder, 'rars',
r.url.split('/')[-1])
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
print('Downloaded %s' % local_filename)
rar = RarFile(local_filename)
rar.extractall(event_folder)
for filename in rar.namelist():
old = os.path.join(event_folder, filename)
new = os.path.join(event_folder,
str(demo['date']) + '#' + filename)
os.rename(old, new)
def __init__(self, archive_path: str):
self.archive_path = archive_path
self.rarfile = RarFile(self.archive_path)
self.namelist = {
i.filename
for i in self.rarfile.infolist() if not i.isdir()
}
def extract(rfile: RarFile, pwd: str) -> bool: # extract函数返回的是bool类型
try:
rfile.extractall(path='.', pwd=pwd.encode('utf-8')) # 密码输入错误的时候会报错
now = time.time() # 故使用 try - except 语句
print(f"Password is: {pwd}") # 将正确的密码输出到控制台
return True
except:
return False
def unrar(self, file_name):
zf = RarFile(join(self.open_path, file_name), 'r')
to_extract = zf.namelist()[0]
zf.extract(
to_extract,
path=self.open_path,
)
return to_extract
def upload_comic(series_id, file=None, file_url=None):
"""
Given a series id and a file or a file url, upload the comic pages
to s3 and create a new Comic instance in the given series.
"""
# We need at least one of the arguments.
if file is None and file_url is None:
return None
# If a file url is provided, download it to memory and get its file name.
if file_url:
req = requests.get(file_url, stream=True)
d = req.headers['content-disposition']
file_name = re.findall("filename=(.+)", d)
file_name = file_name[0][:-1][1:] # Remove double quotes.
file = io.BytesIO(req.content)
closing(req)
# Otherwise simply take its file name.
else:
file_name = file.name
# Determine whether it's a CBR or a CBZ and create a RarFile or a ZipFile.
if file_name.endswith('.{}'.format(CBR)):
cb_file = RarFile(file)
cb_type = CBR
elif file_name.endswith('.{}'.format(CBZ)):
cb_file = ZipFile(file)
cb_type = CBZ
else:
return None
# Go through the CBZ/CBR pages and upload all of them to s3.
page_urls = []
for file_name in cb_file.namelist():
if not file_name.lower().endswith(
'.jpg') and not file_name.lower().endswith('.png'):
continue
image_url = upload(ContentFile(cb_file.read(file_name)),
name=file_name,
prefix=False,
bucket_name=False,
key=None,
secret=None,
host=None,
expires=0,
query_auth=False,
force_http=True,
policy=None)
page_urls.append(image_url)
# Create a comic.
return Comic.objects.create(title=file_name.replace('.cbz', ''),
file_type=cb_type,
pages='|'.join(page_urls),
series_id=series_id)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
raise ProviderError('Unidentified archive type')
# extract subtitle's content
subs_in_archive = []
for name in archive.namelist():
for ext in (".srt", ".sub", ".ssa", ".ass"):
if name.endswith(ext):
subs_in_archive.append(name)
# select the correct subtitle file
matching_sub = None
if len(subs_in_archive) == 1:
matching_sub = subs_in_archive[0]
else:
for sub_name in subs_in_archive:
guess = guessit(sub_name)
# consider subtitle valid if:
# - episode and season match
# - format matches (if it was matched before)
# - release group matches (and we asked for one and it was matched, or it was not matched)
if guess["episode"] == subtitle.episode and guess[
"season"] == subtitle.season:
format_matches = "format" not in subtitle.matches or \
("format" in subtitle.matches and guess["format"].lower() in
subtitle.releases.lower())
release_group_matches = True
if subtitle.asked_for_release_group:
release_group_matches = "release_group" not in subtitle.matches or \
("release_group" in subtitle.matches and
guess["release_group"].lower() ==
subtitle.asked_for_release_group.lower())
if release_group_matches and format_matches:
matching_sub = sub_name
break
if not matching_sub:
raise ProviderError("None of expected subtitle found in archive")
subtitle.content = fix_line_ending(archive.read(matching_sub))
def unrar_files(path_zipped_file, tmp_dir=TEMP_DIR):
"""
Uncompress zipped file (rar extension) by saving in a temporary file.
Args:
path_zipped_file ([str]): respective file path in a rar format.
tmp_dir ([str], optional): temporary file path.
"""
rar = RarFile(path_zipped_file)
temp = tmp_dir
rar.extractall(path=temp)
def __load_archive(self):
try:
self.archive_file = ZipFile(self.binary, 'r')
except BadZipfile:
try:
self.archive_file = RarFile(self.binary, 'r')
except (BadRarFile, NotRarFile, io.UnsupportedOperation,
NeedFirstVolume):
message = str(sys.exc_info()[1])
raise ValueError(f'архив не может быть прочитан ({message})')
return self.archive_file
def unRarFull(fias_object):
"""Распаковка из полной базы ФИАС"""
rf = RarFile(fiases.fias_data.WORK_DIR + fiases.fias_data.FIAS_XML_RAR)
objectMatcher = re.compile(fias_object.FILE)
print('')
for f in rf.infolist():
if objectMatcher.match(f.filename):
fias_object.xml_file = f.filename
fias_object.xml_file_size = f.file_size
if (fias_object.xml_file_size > 0):
rf.extract(fias_object.xml_file, fiases.fias_data.WORK_DIR)
def unrarUpdate(fias_object):
"""Распаковка обновления """
rf = RarFile(fiases.fias_data.WORK_DIR +
fiases.fias_data.FIAS_DELTA_XML_RAR)
fias_objectMatcher = re.compile(fias_object.FILE)
for f in rf.infolist():
if fias_objectMatcher.match(f.filename):
fias_object.xml_delta_file = f.filename
fias_object.xml_delta_file_size = f.file_size
if (fias_object.xml_delta_file_size > 0):
rf.extract(fias_object.xml_delta_file, fiases.fias_data.WORK_DIR)
def rar_archive(self, file_read):
# method for rar archive processing
# the same principle as in zip archive method
done_set = set()
try:
with RarFile(file_read) as my_archive:
for i in my_archive.namelist():
with my_archive.open(i) as myfile:
try:
xml_dict = xmltodict.parse(myfile)
context = my_archive.read(i)
statform = xml_dict['NBUSTATREPORT']['HEAD']['STATFORM']
kod = xml_dict['NBUSTATREPORT']['HEAD']['EDRPOU']
report_date = xml_dict['NBUSTATREPORT']['HEAD']['REPORTDATE']
if statform in self.files_to_save_list and report_date == self.analysis_date:
with open(self.output_path + '/' + statform + '_' + kod + '.xml', 'wb') as output_file:
output_file.write(context)
done_set.add(statform)
except Exception:
try:
with RarFile(myfile) as inner_archive:
for y in inner_archive.namelist():
with inner_archive.open(y) as myfile:
try:
xml_dict = xmltodict.parse(myfile)
context = my_archive.read(y)
statform = xml_dict['NBUSTATREPORT']['HEAD']['STATFORM']
kod = xml_dict['NBUSTATREPORT']['HEAD']['EDRPOU']
report_date = xml_dict['NBUSTATREPORT']['HEAD']['REPORTDATE']
if statform in self.files_to_save_list \
and report_date == self.analysis_date:
with open(self.output_path + '/' + statform + kod + '.xml',
'wb') as output_file:
output_file.write(context)
done_set.add(statform)
except Exception:
pass
except Exception:
pass
except Exception:
pass
for x in self.files_to_save_list:
if x in done_set:
self.ui.textBrowser.append(x + '- OK')
QtWidgets.QApplication.processEvents()
else:
self.ui.textBrowser.append(x + '-' + "<span style=\" font-size:8pt; font-weight:600;"
"color:#ff0000;\" >ERROR!</span>")
QtWidgets.QApplication.processEvents()
def scan_archive(path):
"""Scan an archive from a `path`.
:param str path: existing path to the archive.
:return: the scanned video.
:rtype: :class:`~subliminal.video.Video`
"""
# check for non-existing path
if not os.path.exists(path):
raise ValueError('Path does not exist')
# check video extension
if not path.endswith(ARCHIVE_EXTENSIONS):
raise ValueError('%r is not a valid archive extension' %
os.path.splitext(path)[1])
dirpath, filename = os.path.split(path)
logger.info('Scanning archive %r in %r', filename, dirpath)
# rar extension
if filename.endswith('.rar'):
rar = RarFile(path)
# filter on video extensions
rar_filenames = [
f for f in rar.namelist() if f.endswith(VIDEO_EXTENSIONS)
]
# no video found
if not rar_filenames:
raise ValueError('No video in archive')
# more than one video found
if len(rar_filenames) > 1:
raise ValueError('More than one video in archive')
# guess
rar_filename = rar_filenames[0]
rar_filepath = os.path.join(dirpath, rar_filename)
video = Video.fromguess(rar_filepath, guessit(rar_filepath))
# size
video.size = rar.getinfo(rar_filename).file_size
else:
raise ValueError('Unsupported extension %r' %
os.path.splitext(path)[1])
return video
def run(self):
for idx, file in enumerate(self.input()):
ext = os.path.splitext(file.path)[-1]
if ext == '.rar':
arch = RarFile(file.path)
elif ext == '.zip':
arch = zipfile.ZipFile(file.path)
f = arch.namelist()[0]
target = os.path.join(os.getenv('TEMP_DIR'), f)
arch.extract(f, os.path.join(os.getenv('TEMP_DIR')))
copyfile(
target,
os.path.join(os.getenv('TEMP_DIR'),
os.path.basename(self.output()[idx].path)))
def xtract_rar(inpt: str, password: str = ''): ## {{{
from os import path, mkdir, chdir
from rarfile import RarFile
inpt = remove_trailing_slash(inpt)
root_base, ext = path.splitext(inpt)
dest_dir = root_base
mkdir(dest_dir)
chdir(dest_dir)
if password == '':
with RarFile(inpt, 'r') as CUR_RAR:
CUR_RAR.extractall()
else:
with RarFile(inpt, 'r') as CUR_RAR:
CUR_RAR.extractall(pwd=password)
def unarchive(fname):
ftype = ft.get_type(fname)
if ftype == "rar":
ext = RarFile(fname)
elif ftype == "tar":
ext = tarfile.open(fname)
elif ftype == "zip":
ext = ZipFile(fname)
new_path = get_new_name(fname[:fname.rfind(".")] + "_extracted")
if not os.path.exists(new_path):
os.makedirs(new_path)
ext.extractall(path=new_path)
return new_path
def unarchive(self, fname):
ftype = self.ft.get_type(fname)
if ftype == "rar":
ext = RarFile(fname)
elif ftype == "tar":
ext = tarfile.open(fname)
elif ftype == "zip":
ext = ZipFile(fname)
new_path = fname[:fname.rfind(".")] + "_extracted"
if not os.path.exists(new_path):
os.makedirs(new_path)
ext.extractall(path=new_path)
return new_path
def _init_def(path, capture=_def_formats()):
if path.suffix.lower() in ('.zip', '.cbz'):
o = ZipFile(str(path))
elif path.suffix.lower() in ('.rar', '.cbr'):
o = RarFile(str(path))
o.hpx_path = path
return o
def extrair_arquivos(tipo: str, caminho: str, conteudo):
""" Extrai csvs de um arquivo compactado.
Parâmetros
----------
tipo: str
tipo do arquivo: rar ou zip.
caminho: str
caminho para pasta onde os arquivos devem ser salvos.
conteudo
bytes do arquivo compactado.
"""
if tipo == "x-rar-compressed":
# Salva temporariamente o .rar
with open('file.rar', 'wb') as f:
f.write(conteudo)
# Extrai arquivos csv
with RarFile("file.rar") as rf:
extrair_rar(rf, caminho)
# Apaga .rar
os.remove('file.rar')
else:
zipfile = ZipFile(BytesIO(conteudo))
with zipfile as zp:
extrair_zip(zp, caminho)
def extract_by_fileslist(apath, files_list):
"""
Extract files from archive. Supports only rar, zip archives.
"""
# identify format
frmt = identify_file_format(apath)
_files_list, arch_obj = [], None
if not frmt:
raise ExternalSourceError("Not supported format")
else:
if frmt == 'zip':
arch_obj = ZipFile(apath)
elif frmt == 'rar':
arch_obj = RarFile(apath)
paths = []
for f in files_list:
folder = os.path.abspath(os.path.dirname(f))
fname = os.path.basename(f)
for _f in arch_obj.namelist():
if os.path.basename(_f) == fname:
arch_obj.extract(_f, folder)
# build path for just new extracted file
src = os.path.join(folder, _f).replace('/', os.sep)
move(src, f)
paths.append(f)
return paths
def extract_rar(complete_folder, rar_file, rar_name, data_name):
"""
Extract rar.gz file and get the data.
"""
rar_file_path = os.path.join(complete_folder, rar_file)
rf = RarFile(rar_file_path)
rar_inside = rf.namelist()
for t_i in rar_inside:
if rar_name in t_i:
rf.extract(t_i, path=complete_folder)
shutil.move(os.path.join(complete_folder, t_i), os.path.join(complete_folder, data_name))
break
try:
shutil.rmtree(os.path.join(complete_folder, t_i.split('/')[0]))
except:
pass
def __init__(self, shuffled=False):
"""
As the data needs to be downloaded from the website, the initialiser handles that as well,
then calls _load_all_data to handle loading the contents of the downloaded data.
:param shuffled: boolean
Pass true to shuffle the dataset.
This parameter is not very meaningful in this case because _load_all_data shuffles
the data anyway.
"""
self.data_length = 200
self.training_fraction = 0.2
expected_shape = (int(self.data_length / 2), 4)
self._compressed_data_path = "ActivityData.rar"
self._uncompressed_data_path = "DataSet/"
self._all_data = None
self._all_data_path = "all_data" + type(self).__name__ + ".npy"
self._all_labels_path = "all_labels" + type(self).__name__ + ".npy"
if os.path.isfile(self._all_data_path):
self._all_data = (np.load(self._all_data_path),
np.load(self._all_labels_path))
if self._all_data[0][0].shape != expected_shape:
self._all_data = None
if not os.path.isfile(self._compressed_data_path):
urllib.request.urlretrieve(
"http://ps.ewi.utwente.nl/Blog/Sensors_Activity_Recognition_DataSet_Shoaib.rar",
self._compressed_data_path)
if not os.path.isfile(self._uncompressed_data_path):
# REMEMBER THAT THIS REQUIRES UNRAR INSTALLED
with RarFile("ActivityData.rar") as rf:
rf.extractall()
super().__init__(expected_shape, shuffled)
def download_archive_and_add_subtitle_files(self, link, language, video,
fps, num_cds):
logger.info('Downloading subtitle %r', link)
cache_key = sha1(link.encode("utf-8")).digest()
request = region.get(cache_key)
if request is NO_VALUE:
request = self.session.get(
link, headers={'Referer': 'https://subsunacs.net/search.php'})
request.raise_for_status()
region.set(cache_key, request)
else:
logger.info('Cache file: %s',
codecs.encode(cache_key, 'hex_codec').decode('utf-8'))
try:
archive_stream = io.BytesIO(request.content)
if is_rarfile(archive_stream):
return self.process_archive_subtitle_files(
RarFile(archive_stream), language, video, link, fps,
num_cds)
elif is_zipfile(archive_stream):
return self.process_archive_subtitle_files(
ZipFile(archive_stream), language, video, link, fps,
num_cds)
elif archive_stream.seek(0) == 0 and is_7zfile(archive_stream):
return self.process_archive_subtitle_files(
SevenZipFile(archive_stream), language, video, link, fps,
num_cds)
except:
pass
logger.error('Ignore unsupported archive %r', request.headers)
region.delete(cache_key)
return []
def crackrar(filepath, passwd):
i = 0
with RarFile(filepath, 'r') as rarObj:
for password in passwd:
i += 1
try:
rarObj.extractall(path='./extractedfile',
members=None,
pwd=password.rstrip('\n'))
print('\n===================')
print('rar file extracted!')
print('===================\n')
requests.get(
f'http://localhost:5000/recieve?status=extracted&password={password}&processid={processid}'
)
return ''
except:
pass
try:
if i % 20 == 0:
requests.get(
f'http://localhost:5000/progress?processid={processid}&numpasswords={numpasswords}&testedpasswords={i}'
)
except Exception as e:
requests.get(f'http://localhost:5000/errors?error={e}')
requests.get('http://localhost:5000/recieve?status=failed')
requests.get(
f'http://localhost:5000/progress?processid={processid}&numpasswords={numpasswords}&testedpasswords={i}'
)
def get_cb_file_for_comic(comic: models.FileItem) -> Union[ZipFile, RarFile]:
if _is_file_name_cbz(comic.name):
return ZipFile(comic.path)
elif _is_file_name_cbr(comic.name):
return RarFile(comic.path)
else:
raise TypeError(f"Unable to get a cb file for {comic}")
def extract_rar(self, rar_path, extract_path, password):
"""Extracts a nested RAR file.
@param rar_path: RAR path
@param extract_path: where to extract
@param password: RAR password
"""
# Test if rar file contains a file named as itself.
if self.is_overwritten(rar_path):
log.debug(
"RAR file contains a file with the same name, original is going to be overwrite"
)
# TODO: add random string.
new_rar_path = rar_path + ".old"
shutil.move(rar_path, new_rar_path)
rar_path = new_rar_path
# Extraction.
with RarFile(rar_path, "r") as archive:
try:
archive.extractall(path=extract_path, pwd=password)
except BadRarFile:
raise CuckooPackageError("Invalid Rar file")
except RuntimeError:
try:
archive.extractall(path=extract_path, pwd="infected")
except RuntimeError as e:
raise CuckooPackageError("Unable to extract Rar file: "
"{0}".format(e))
finally:
# Extract nested archives.
for name in archive.namelist():
if name.endswith(".rar"):
# Recurse.
self.extract_rar(os.path.join(extract_path, name),
extract_path, password)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link,
headers={'Referer': self.api_url},
timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
subtitle.content = r.content
if subtitle.is_valid():
return
subtitle.content = None
raise ProviderError('Unidentified archive type')
if subtitle.is_episode:
subtitle.content = self._get_subtitle_from_archive(
subtitle, archive)
else:
subtitle.content = self.get_subtitle_from_archive(
subtitle, archive)
def extract(entry, name):
rfile = RarFile(localsettings.BASEDIR + name)
start = datetime.now().replace(microsecond=0)
try:
extdir = localsettings.OUTDIR
if entry[-4:] == ".rar":
extdir = localsettings.BASEDIR + RARTEMP
rfile.extract(entry, path=extdir)
print name, "extracted"
except:
status = 'failed'
print name, "failed extract"
response = jsonify(status=status)
response.status_code = 400
return response
time = str(datetime.now().replace(microsecond=0) - start)
return jsonify(time=time, file=entry, status='success')
def scan_archive(path):
"""Scan an archive from a `path`.
:param str path: existing path to the archive.
:return: the scanned video.
:rtype: :class:`~subliminal.video.Video`
"""
# check for non-existing path
if not os.path.exists(path):
raise ValueError('Path does not exist')
# check video extension
if not path.endswith(ARCHIVE_EXTENSIONS):
raise ValueError('%r is not a valid archive extension' % os.path.splitext(path)[1])
dirpath, filename = os.path.split(path)
logger.info('Scanning archive %r in %r', filename, dirpath)
# rar extension
if filename.endswith('.rar'):
rar = RarFile(path)
# filter on video extensions
rar_filenames = [f for f in rar.namelist() if f.endswith(VIDEO_EXTENSIONS)]
# no video found
if not rar_filenames:
raise ValueError('No video in archive')
# more than one video found
if len(rar_filenames) > 1:
raise ValueError('More than one video in archive')
# guess
rar_filename = rar_filenames[0]
rar_filepath = os.path.join(dirpath, rar_filename)
video = Video.fromguess(rar_filepath, guessit(rar_filepath))
# size
video.size = rar.getinfo(rar_filename).file_size
else:
raise ValueError('Unsupported extension %r' % os.path.splitext(path)[1])
return video
class ArchivedFile(object):
def __init__(self,filename):
if is_zipfile(filename):
self._zfile = ZipFile(filename)
elif is_rarfile(filename):
self._zfile = RarFile(filename)
def filelist(self):
return self._zfile.namelist()
def extract(self,srcfile,targetpath):
"extract srcfile in archive to targetpath"
for fullfilename in self.filelist():
if srcfile in fullfilename:
fpath,fname = os.path.split(fullfilename)
self._zfile.extract(fname.encode('gbk'),targetpath+fname)
return True
return None
def extractall(self,targetpath):
self._zfile.extractall(targetpath)
class RarArch( Arch ):
def __init__( self, name, path = None ):
Arch.__init__( self, name , path )
self.open()
def open( self ):
from rarfile import RarFile
self.arch_desc = RarFile( self.name, 'r' )
def name_list( self ):
for item in self.arch_desc.infolist():
if not item.isdir():
yield item.filename
def extract_rar(self,tarfile_fullpath, delete_tar_file=DELETE,Path=True):
tarfile_name = os.path.basename(tarfile_fullpath)
parent_dir = os.path.dirname(tarfile_fullpath)
extract_folder_name = tarfile_name[:-1*len(self.fileExtension(tarfile_name))-1]
if Path:
if self.extract_path:
extract_folder_fullpath = os.path.join(self.extract_path, extract_folder_name)
else:
extract_folder_fullpath = os.path.join(parent_dir,"archivedFiles", extract_folder_name)
else:
extract_folder_fullpath = os.path.join(parent_dir, extract_folder_name)
try:
rar = RarFile(tarfile_fullpath, "r")
rar.extractall(extract_folder_fullpath)
rar.close()
if delete_tar_file:
os.remove(tarfile_fullpath)
return extract_folder_fullpath
except Exception as e:
# Exceptions can occur while opening a damaged tar file.
print 'Error occured while extracting %s\n'\
'Reason: %s' %(tarfile_fullpath, e)
return
def download_subtitle(self, subtitle):
logger.info("Download subtitle %r", subtitle.link)
r = self.session.get(subtitle.link, params=None, timeout=10)
r.raise_for_status()
# check if it's rar or zip
if r.content[:4] == "Rar!":
compressed = RarFile(StringIO(r.content))
content = compressed.read(compressed.namelist()[0])
elif r.content[:4] == "PK":
compressed = ZipFile(StringIO(r.content))
content = compressed.read(compressed.namelist()[0])
else:
content = r.content
# some ppl just put wrong file inside
if mimetypes.guess_type(content)[0] != "text/plain":
raise ProviderError("Wrong subtitle inside compressed file")
subtitle.content = fix_line_ending(content)
from rarfile import RarFile
from rarfile import BadRarFile
from rarfile import PasswordRequired
import os
import threading
lock=threading.RLock()
rar=RarFile("/home/ljd/1.rar")
filename=rar.namelist()[0]
def loopkey(start,stop):
for x in xrange(start,stop):
try:
rar.setpassword(str(x))
rar.read(filename)
print x
os._exit(0)
except BadRarFile,PasswordRequired:
pass
def looprange(n):
data=range(0,n+1,n/100)
return [(value,data[index+1]) for index,value in enumerate(data[:-1])]
data=looprange(1000000000)
print data
for x,y in data:
threading.Thread(target=loopkey,args=(x,y)).start()
ret = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError:
log.error("Error on Find with: %s" % ret)
# str_err = "Error on Find command, rerun" + torrent_name
# push = pb.push_note("[Tor Complete]",str_err)
sys.exit(-2)
ret = ret.rstrip()
log.debug("Find Returned : %s" % ret)
# cmd = 'mkdir "'+STAGING_PATH+path+torrent_id+'/" && cd "'+STAGING_PATH+path+torrent_id+'/" && easy_extract --force '+root
# log.debug('Shelling out : %s' % cmd)
# ret = subprocess.call(cmd, shell=True)
# THIS IS THE RARFILE method
rar_file = ret
out_path = STAGING_PATH + path + torrent_id + "/"
log.debug("file=%s dest=%s" % (rar_file, out_path))
rf = RarFile(rar_file)
try:
rf.extractall(path=out_path)
unrar_success = True
except:
e = sys.exc_info()[0]
log.error("Error on Extraction with %s" % e)
# str_err = "Error on Extraction, error: " + e
# push = pb.push_note("[Tor Complete]",str_err)
sys.exit(-2)
# with RarFile(rar_file,path=out_path) as rf:
# rf.extractall()
# cmd='/home/bsmith/src/unrar/unrar x -o+ "'+ret+'" "'+STAGING_PATH+path+torrent_id+'/"'
# log.debug('Shelling out : %s' % cmd)
# ret = subprocess.call(cmd, shell=True)
# if ret != 0:
def open( self ):
from rarfile import RarFile
self.arch_desc = RarFile( self.name, 'r' )
def __init__(self,fname): #RarFile.__init__(self,fname,info_callback=self.rar_dirs) RarFile.__init__(self,fname)
def unrar(path, rar_files, force, result): # pylint: disable=too-many-branches,too-many-statements
"""
Extracts RAR files
:param path: Path to look for files in
:param rar_files: Names of RAR files
:param force: process currently processing items
:param result: Previous results
:return: List of unpacked file names
"""
unpacked_dirs = []
if sickbeard.UNPACK == 1 and rar_files:
result.output += log_helper("Packed Releases detected: {0}".format(rar_files), logger.DEBUG)
for archive in rar_files:
failure = None
rar_handle = None
try:
archive_path = ek(os.path.join, path, archive)
if already_processed(path, archive, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(archive_path), logger.DEBUG)
continue
if not helpers.is_rar_file(archive_path):
continue
result.output += log_helper("Checking if archive is valid and contains a video: {0}".format(archive_path), logger.DEBUG)
rar_handle = RarFile(archive_path)
if rar_handle.needs_password():
# TODO: Add support in settings for a list of passwords to try here with rar_handle.set_password(x)
result.output += log_helper('Archive needs a password, skipping: {0}'.format(archive_path))
continue
# rar_handle.testrar()
# If there are no video files in the rar, don't extract it
rar_media_files = filter(helpers.is_media_file, rar_handle.namelist())
if not rar_media_files:
continue
rar_release_name = archive.rpartition('.')[0]
# Choose the directory we'll unpack to:
if sickbeard.UNPACK_DIR and os.path.isdir(sickbeard.UNPACK_DIR): # verify the unpack dir exists
unpack_base_dir = sickbeard.UNPACK_DIR
else:
unpack_base_dir = path
if sickbeard.UNPACK_DIR: # Let user know if we can't unpack there
result.output += log_helper('Unpack directory cannot be verified. Using {0}'.format(path), logger.DEBUG)
# Fix up the list for checking if already processed
rar_media_files = [os.path.join(unpack_base_dir, rar_release_name, rar_media_file) for rar_media_file in rar_media_files]
skip_rar = False
for rar_media_file in rar_media_files:
check_path, check_file = os.path.split(rar_media_file)
if already_processed(check_path, check_file, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(rar_media_file), logger.DEBUG)
skip_rar = True
break
if skip_rar:
continue
rar_extract_path = ek(os.path.join, unpack_base_dir, rar_release_name)
result.output += log_helper("Unpacking archive: {0}".format(archive), logger.DEBUG)
rar_handle.extractall(path=rar_extract_path)
unpacked_dirs.append(rar_extract_path)
except RarCRCError:
failure = ('Archive Broken', 'Unpacking failed because of a CRC error')
except RarWrongPassword:
failure = ('Incorrect RAR Password', 'Unpacking failed because of an Incorrect Rar Password')
except PasswordRequired:
failure = ('Rar is password protected', 'Unpacking failed because it needs a password')
except RarOpenError:
failure = ('Rar Open Error, check the parent folder and destination file permissions.',
'Unpacking failed with a File Open Error (file permissions?)')
except RarExecError:
failure = ('Invalid Rar Archive Usage', 'Unpacking Failed with Invalid Rar Archive Usage. Is unrar installed and on the system PATH?')
except BadRarFile:
failure = ('Invalid Rar Archive', 'Unpacking Failed with an Invalid Rar Archive Error')
except NeedFirstVolume:
continue
except (Exception, Error) as e:
failure = (ex(e), 'Unpacking failed')
finally:
if rar_handle:
del rar_handle
if failure:
result.output += log_helper('Failed to extract the archive {0}: {1}'.format(archive, failure[0]), logger.WARNING)
result.missed_files.append('{0} : Unpacking failed: {1}'.format(archive, failure[1]))
result.result = False
continue
return unpacked_dirs
def handle(self, *args, **options):
update_flag = True
try:
latest_update = FiasUpdateInfo.objects.latest("create_ts")
except FiasUpdateInfo.DoesNotExist:
latest_update = None
fias_soap = Client(settings.FIAS_URL)
latest_soap = fias_soap.service.GetLastDownloadFileInfo()
version = latest_soap.VersionId
if latest_update:
if int(version) <= int(latest_update.version):
update_flag = False
if update_flag:
xurl = latest_soap.FiasDeltaXmlUrl
delta_file = urllib2.urlopen(xurl)
input_file = StringIO(delta_file.read())
new_update = FiasUpdateInfo(version=version)
new_update.textversion = latest_soap.TextVersion.encode("utf8")
new_update.delta_url = latest_soap.FiasDeltaXmlUrl
new_update.delta_file.save("fias_update_%s.rar" % version, ContentFile(input_file.getvalue()), save=False)
new_update.save()
# unpack, get xml, write to DB
rar_file = RarFile(new_update.delta_file.path)
update_file_addr = None
update_file_house = None
for packed_file in rar_file.namelist():
if packed_file.find("_ADDROBJ_") >= 0:
update_file_addr = packed_file
if packed_file.find("_HOUSE_") >= 0:
update_file_house = packed_file
# AddressObj
if not update_file_addr:
xml_string_addr = rar_file.read(update_file_addr)
xml_tree_addr = etree.fromstring(xml_string_addr)
update_items_addr = xml_tree_addr.getchildren()
if update_items_addr and update_items_addr[0].keys():
fields_addr = update_items_addr[0].keys()
update_length_addr = len(update_items_addr)
for counter_addr, update_item_addr in enumerate(update_items_addr):
new_addrobj = AddressObj()
for field_addr in fields_addr:
setattr(new_addrobj, field_addr, update_item_addr.get(field_addr))
new_addrobj.save()
print u"%s Address objects left" % (update_length_addr - counter_addr)
else:
print u"Wrong format of Address update file"
else:
print u"AddressObj file not found in the update"
# House
if update_file_house:
xml_string_house = rar_file.read(update_file_house)
xml_tree_house = etree.fromstring(xml_string_house)
update_items_house = xml_tree_house.getchildren()
if update_items_house and update_items_house[0].keys():
fields_house = update_items_house[0].keys()
update_length_house = len(update_items_house)
for counter_house, update_item_house in enumerate(update_items_house):
new_house = House()
for field_house in fields_house:
setattr(new_house, field_house, update_item_house.get(field_house))
new_house.save()
print u"%s House objects left" % (update_length_house - counter_house)
else:
print u"Wrong format of House update file"
else:
print u"House file not found in the update"
print u"Updated successfully"
else:
print u"No new updates found"
def add(self, archive_password=None, audio_file=None, session=None, artist_name_fallback=None):
cache_key = LibraryUpload.CACHE_KEY % (cherrypy.request.user.id, session)
all_tracks = None
if not cache.has(cache_key):
raise cherrypy.HTTPError(status=409)
else:
all_tracks = cache.get(cache_key)
if audio_file is not None and len(audio_file) == 0:
audio_file = None
if archive_password is not None and len(archive_password) == 0:
archive_password = None
content_disposition = cherrypy.request.headers.get('content-disposition')
filename = content_disposition[content_disposition.index('filename=') + 9:]
if filename.startswith('"') and filename.endswith('"'):
filename = filename[1:-1]
filename = unquote(filename)
ext = os.path.splitext(filename)[1].lower()[1:]
basename = os.path.splitext(filename)[0]
cache_path = os.path.join(cherrypy.config['opmuse'].get('cache.path'), 'upload')
if not os.path.exists(cache_path):
os.mkdir(cache_path)
tempdir = tempfile.mkdtemp(dir=cache_path)
path = os.path.join(tempdir, filename)
paths = []
rarfile.PATH_SEP = '/'
messages = []
with open(path, 'wb') as fileobj:
fileobj.write(cherrypy.request.rfile.read())
# this file is a regular file that belongs to an audio_file
if audio_file is not None:
track = None
tries = 0
# try and sleep until we get the track.. this will almost always
# be needed because of the async upload.
while track is None and tries < 10:
track = library_dao.get_track_by_filename(audio_file.encode('utf8'))
tries += 1
if track is None:
time.sleep(3)
if track is None:
messages.append(('warning', ("<strong>%s</strong>: Skipping <strong>%s</strong>, timeout trying to " +
"find its track.") % (audio_file, filename)))
else:
track_structure = TrackStructureParser(track)
track_path = track_structure.get_path(absolute=True)
relative_track_path = track_structure.get_path(absolute=False).decode('utf8', 'replace')
new_path = os.path.join(track_path, filename.encode('utf8'))
if os.path.exists(new_path):
messages.append(('warning', ("<strong>%s</strong>: Skipping <strong>%s</strong>, already exists " +
"in <strong>%s</strong>.") % (audio_file, filename, relative_track_path)))
else:
shutil.move(path.encode('utf8'), new_path)
messages.append(('info', ("<strong>%s</strong>: Uploaded <strong>%s</strong> to " +
"<strong>%s</strong>.") % (audio_file, filename, relative_track_path)))
elif ext == "zip":
# set artist name fallback to zip's name so if it's missing artist tags
# it's easily distinguishable and editable so it can be fixed after upload.
artist_name_fallback = basename
try:
zip = ZipFile(path)
if archive_password is not None:
zip.setpassword(archive_password.encode())
zip.extractall(tempdir)
os.remove(path)
for name in zip.namelist():
namepath = os.path.join(tempdir, name)
# ignore hidden files, e.g. OSX archive weirdness and such
if name.startswith(".") or os.path.split(name)[0] == "__MACOSX":
shutil.rmtree(namepath)
continue
paths.append(namepath.encode('utf8'))
except Exception as error:
messages.append(('danger', "<strong>%s</strong>: %s" % (os.path.basename(path), error)))
elif ext == "rar":
# look at corresponding ext == zip comment...
artist_name_fallback = basename
try:
rar = RarFile(path)
if archive_password is None and rar.needs_password():
messages.append(('danger', "<strong>%s</strong>: Needs password but none provided." %
os.path.basename(path)))
else:
if archive_password is not None:
rar.setpassword(archive_password)
rar.extractall(tempdir)
os.remove(path)
for name in rar.namelist():
namepath = os.path.join(tempdir, name)
if name.startswith("."):
shutil.rmtree(namepath)
continue
paths.append(namepath.encode('utf8'))
except Exception as error:
messages.append(('danger', "<strong>%s</strong>: %s" % (os.path.basename(path), error)))
# this is a plain audio file
else:
paths.append(path.encode('utf8'))
for path in paths:
# update modified time to now, we don't want the time from the zip
# archive or whatever
os.utime(path, None)
if len(paths) > 0:
tracks, add_files_messages = library_dao.add_files(paths, move=True, remove_dirs=False,
artist_name_fallback=artist_name_fallback,
user=cherrypy.request.user)
messages += add_files_messages
else:
tracks = []
shutil.rmtree(tempdir)
for track in tracks:
all_tracks.append(track.id)
if track.album is not None:
remotes.update_album(track.album)
if track.artist is not None:
remotes.update_artist(track.artist)
remotes.update_track(track)
hierarchy = Library._produce_track_hierarchy(library_dao.get_tracks_by_ids(all_tracks))
return {'hierarchy': hierarchy, 'messages': messages}
def __init__(self,filename):
if is_zipfile(filename):
self._zfile = ZipFile(filename)
elif is_rarfile(filename):
self._zfile = RarFile(filename)
from rarfile import RarFile
from rarfile import BadRarFile
from rarfile import PasswordRequired
rar=RarFile("/home/ljd/1.rar")
filename=rar.namelist()[0]
for x in xrange(100000):
try:
rar.setpassword(str(x))
rar.read(filename)
print x
exit()
except BadRarFile,PasswordRequired:
pass
def unrar( filename, destination=None, report=False ):
from rarfile import RarFile
base_dir = ""
if destination is None:
destination = os.path.dirname( filename )
try:
rar = RarFile( filename, "r" )
namelist = rar.namelist()
total_items = len( namelist ) or 1
diff = 100.0 / total_items
percent = 0
# nom du dossier racine
root_dir = namelist[ -1 ]
is_root_dir = True
# si root_dir n'est pas un dossier ou n'est pas la racine, on se base sur le nom de l'archive
#print root_dir
if not rar.getinfo( root_dir ).isdir():
is_root_dir = False
else:
for i in namelist:
#print root_dir in i, i
if not root_dir in i:
is_root_dir = False
break
if not is_root_dir:#rar.getinfo( root_dir ).isdir():
root_dir = os.path.basename( os.path.splitext( filename )[ 0 ] )
base_dir = os.path.join( destination, root_dir )
if os.path.isdir( base_dir ):
shutil2.rmtree( base_dir )
os.makedirs( base_dir )
time_sleep = get_time_sleep( filename )
# avec cette methode on extract dans le dossier ou est l'archive
ok = executebuiltin( 'XBMC.Extract(%s)' % ( filename, ) )
#sleep une seconde ou plus selon la drosseur du fichier le temps que le builtin est fini car l'action suivante
# "os.listdir" est excecuter avant la fin du builtin
sleep( time_sleep )
# si le dossier base_dir est vide on move les items de namelist dedans
if not os.listdir( base_dir ):
for item in namelist:
src = os.path.normpath( os.path.join( os.path.dirname( filename ), item ) )
dst = os.path.normpath( os.path.join( base_dir, item ) )
if not rar.getinfo( item ).isdir():
if not os.path.isdir( os.path.dirname( dst ) ):
os.makedirs( os.path.dirname( dst ) )
shutil2.move( src, dst, overwrite=True )
elif os.path.exists( src ) and not os.listdir( src ):
shutil2.rmtree( src )
#maintenant on verifier l'extraction d'xbmc avec la liste de la lib rarfile
if os.path.isdir( base_dir ):
size = 0
list_size = 0
if not root_dir in namelist:
list_size -= 1
namelist = [ os.path.split( item )[ 1 ] for item in namelist ]
for root, dirs, files in os.walk( base_dir, topdown=False ):
percent += diff
list_size += 1
for file in files:
percent += diff
list_size += 1
if report:
if DIALOG_PROGRESS.iscanceled():
break
DIALOG_PROGRESS.update( int( percent ), Language( 187 ) % ( list_size, total_items ), file, Language( 110 ) )
#print round( percent, 2 ), file
if file in namelist:
size += os.path.getsize( os.path.join( root, file ) )
else:
print "Error %s est dans la liste de depart!" % file
#print size
if not size:
print "Error for extracting rar: %s" % filename
rar.close()
del rar
# si list_size est pas declarer une erreur automatique est creer ;)
return base_dir, list_size == total_items
except:
print_exc()
return "", False
