class RarArchive(BaseArchive):
def __init__(self, file):
self._archive = RarFile(file)
def namelist(self):
return self._archive.namelist()
def filenames(self):
return self._archive.namelist()
def close(self):
return self._archive.close()
def is_encrypted(self):
return self._archive.needs_password()
def extraxt(self):
rarfile = rarfile.RarFile.self._archive
rarfile.extraxt(self._archive._rarfile)
def extractall(self, file):
return self._archive.extractall(file)
def list(self, *args, **kwargs):
self.printdir(*args, **kwargs)
def printdir(self, file=None):
"""Print a table of contents for the zip file."""
print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"),
file=file)
for file_ in self._archive._file_parser.infolist():
date = "%d-%02d-%02d %02d:%02d:%02d" % file_.date_time[:6]
print("%-46s %s %12d" % (file_.filename, date, file_.file_size),
file=file)
def download_demo(demo, event_folder):
print('Downloading %s' % demo['url'])
with requests.get(demo['url'],
allow_redirects=not DEBUG,
headers={'User-Agent': 'joder'},
stream=True) as r:
r.raise_for_status()
if DEBUG:
print(r.headers['Location'])
else:
local_filename = os.path.join(event_folder, 'rars',
r.url.split('/')[-1])
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
print('Downloaded %s' % local_filename)
rar = RarFile(local_filename)
rar.extractall(event_folder)
for filename in rar.namelist():
old = os.path.join(event_folder, filename)
new = os.path.join(event_folder,
str(demo['date']) + '#' + filename)
os.rename(old, new)
def each(self, target):
tmpdir = tempdir()
rf = RarFile(target)
namelist = rf.namelist()
for name in namelist:
try:
rf.extract(name, tmpdir)
filepath = os.path.join(
tmpdir,
name,
)
if os.path.isfile(filepath):
self.add_extracted_file(filepath)
except RuntimeError:
for password in ['virus', 'infected']:
try:
filepath = rf.extract(name, tmpdir, pwd=password)
if os.path.isfile(filepath):
self.add_extracted_file(filepath)
break
except RuntimeError:
pass
else:
self.log('error', 'Could not extract {}'.format(name))
return True
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
subtitle.content = r.content
if subtitle.is_valid():
return
subtitle.content = None
raise ProviderError('Unidentified archive type')
subs_in_archive = archive.namelist()
# if Serbian lat and cyr versions are packed together, try to find right version
if len(subs_in_archive) > 1 and (subtitle.language == 'sr' or subtitle.language == 'sr-Cyrl'):
self.get_subtitle_from_bundled_archive(subtitle, subs_in_archive, archive)
else:
# use default method for everything else
subtitle.content = self.get_subtitle_from_archive(subtitle, archive)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
subtitle.content = r.content
if subtitle.is_valid():
return
subtitle.content = None
raise ProviderError('Unidentified archive type')
subs_in_archive = archive.namelist()
# if Serbian lat and cyr versions are packed together, try to find right version
if len(subs_in_archive) > 1 and (subtitle.language == 'sr'
or subtitle.language == 'sr-Cyrl'):
self.get_subtitle_from_bundled_archive(subtitle, subs_in_archive,
archive)
else:
# use default method for everything else
subtitle.content = self.get_subtitle_from_archive(
subtitle, archive)
def __append_tables_info(self, archive_file, inner_file_name,
inner_file_ext):
try:
inner_file = archive_file.read(inner_file_name)
binary = io.BytesIO(inner_file).read()
except (BadZipfile, BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError(f'{inner_file_ext}-архив не может быть прочитан')
if inner_file_ext == 'zip':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
zip_file = ZipFile(file_read, 'r')
for file_name in zip_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(zip_file, file_name, file_ext)
except BadZipfile:
raise TypeError('zip-архив не может быть прочитан')
elif inner_file_ext == 'rar':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
rar_file = RarFile(file_read)
for file_name in rar_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(rar_file, file_name, file_ext)
except (BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError('rar-архив не может быть прочитан')
parser = self.__choose_parser(inner_file_ext)(binary=binary, html=None)
tables_info = parser.get_tables_info()
for ti in tables_info:
self.tables_info.append(ti)
def save_file(self, datafile):
"""
Ouvrir le fichier uploadé et créer les images contenues
:param datafile: nom du fichier d'archive ou handle de fichier
"""
if isinstance(datafile, str):
datafile = open(datafile, 'r')
content_type = datafile.content_type
if content_type in {
'application/zip', 'application/x-zip-compressed',
'application/x-rar-compressed'
}:
if content_type in {'application/x-rar-compressed'}:
archive = RarFile(datafile, 'r')
else:
archive = ZipFile(datafile, 'r')
names = archive.namelist()
for name in names:
filename, fileext = os.path.splitext(name.lower())
if fileext in ('.png', '.jpg', '.jpeg'):
item = archive.open(name)
with NamedTemporaryFile(prefix=slugify(filename),
suffix=fileext,
delete=False) as tfile:
tfile.write(item.read())
picture = Picture(author=self.request.user)
picture.image.save(tfile.name, File(tfile))
picture.save()
item.close()
archive.close()
datafile.close()
return self.cleaned_data
else:
raise forms.ValidationError(_("File must be a zip or rar file"))
def download_subtitle(self, subtitle):
logger.info('Downloading archive %s', subtitle)
r = self.session.get(subtitle.subtitle_url, headers={'Referer': MAIN_SUBDIVX_URL+subtitle.subtitle_id},
timeout=10, verify=True)
r.raise_for_status()
# open the archive
content = None
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Identified rar archive')
content = RarFile(archive_stream)
# logger.info('RarFile archive %r', content)
elif is_zipfile(archive_stream):
logger.debug('Identified zip archive')
content = ZipFile(archive_stream)
else:
raise ValueError('Not a valid archive')
# TODO
content_list = content.namelist()
# NON_LATINO_REFERENCES_IN_FILENAME = ['Espa§a'.decode('utf-8'),'espa§a'.decode('utf-8')]
NON_LATINO_REFERENCES_IN_FILENAME = ['Espa§a', 'espa§a']
# logger.info('archive content_list %r', content_list)
if len(content_list) == 1:
sub = fix_line_ending(content.read(content_list[0]))
else:
for name in content_list:
# logger.debug('name archive')
logger.debug('name archive %s', name)
# discard thae FORZADOS file
if name.endswith('FORZADO.srt'):
logger.debug('name.endswith(FORZADO.srt): %s', name)
continue
# discard hidden files
if os.path.split(name)[-1].startswith('.'):
logger.debug(
'os.path.split(name)[-1].startswith(.): %s', name)
continue
# LatinoamÇrica Espa§a
# discard non-subtitle files
if not name.lower().endswith(MY_SUBTITLE_EXTENSIONS):
logger.debug(
'not name.lower().endswith(SUBTITLE_EXTENSIONS): %s', name)
continue
# discard Espa§a subtitle files
if any(word in name for word in NON_LATINO_REFERENCES_IN_FILENAME):
logger.debug('discard España subtitle files')
continue
else:
logger.debug('sub selected: %s', name)
sub = fix_line_ending(content.read(name))
# logger.info('sub %r', sub)
subtitle.content = sub
def unrar(self, file_name):
zf = RarFile(join(self.open_path, file_name), 'r')
to_extract = zf.namelist()[0]
zf.extract(
to_extract,
path=self.open_path,
)
return to_extract
def extract_by_wildcard(arch_fpath: str,
directory: str = None,
wildcard: str = '*.xlsx',
names=None):
""" Extract files from archive. Supports only zip and rar formats. """
frmt = identify_file_format(arch_fpath)
# detect archive format
if not frmt:
raise ExternalSourceError("Not supported format")
else:
if frmt == 'rar':
arch_obj = RarFile(arch_fpath)
else:
arch_obj = ZipFile(arch_fpath)
if directory:
_dir = directory
else:
_dir = os.path.abspath(os.path.dirname(arch_fpath))
# filter by wildcard
_flist = fnmatch.filter(arch_obj.namelist(), wildcard)
if names:
_flist = _flist[:len(names)]
extracted_files_list = []
# extracting
for i, f in enumerate(_flist):
_fname = os.path.basename(f)
for _f in arch_obj.namelist():
if os.path.basename(_f) == _fname:
arch_obj.extract(_f, _dir)
src = os.path.join(_dir, _f).replace('/', os.sep)
dest = os.path.join(_dir, _fname)
if names:
dest = os.path.join(_dir, names[i])
if _fname:
move(src, dest)
extracted_files_list.append(dest)
return extracted_files_list
def upload_comic(series_id, file=None, file_url=None):
"""
Given a series id and a file or a file url, upload the comic pages
to s3 and create a new Comic instance in the given series.
"""
# We need at least one of the arguments.
if file is None and file_url is None:
return None
# If a file url is provided, download it to memory and get its file name.
if file_url:
req = requests.get(file_url, stream=True)
d = req.headers['content-disposition']
file_name = re.findall("filename=(.+)", d)
file_name = file_name[0][:-1][1:] # Remove double quotes.
file = io.BytesIO(req.content)
closing(req)
# Otherwise simply take its file name.
else:
file_name = file.name
# Determine whether it's a CBR or a CBZ and create a RarFile or a ZipFile.
if file_name.endswith('.{}'.format(CBR)):
cb_file = RarFile(file)
cb_type = CBR
elif file_name.endswith('.{}'.format(CBZ)):
cb_file = ZipFile(file)
cb_type = CBZ
else:
return None
# Go through the CBZ/CBR pages and upload all of them to s3.
page_urls = []
for file_name in cb_file.namelist():
if not file_name.lower().endswith(
'.jpg') and not file_name.lower().endswith('.png'):
continue
image_url = upload(ContentFile(cb_file.read(file_name)),
name=file_name,
prefix=False,
bucket_name=False,
key=None,
secret=None,
host=None,
expires=0,
query_auth=False,
force_http=True,
policy=None)
page_urls.append(image_url)
# Create a comic.
return Comic.objects.create(title=file_name.replace('.cbz', ''),
file_type=cb_type,
pages='|'.join(page_urls),
series_id=series_id)
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
raise ProviderError('Unidentified archive type')
# extract subtitle's content
subs_in_archive = []
for name in archive.namelist():
for ext in (".srt", ".sub", ".ssa", ".ass"):
if name.endswith(ext):
subs_in_archive.append(name)
# select the correct subtitle file
matching_sub = None
if len(subs_in_archive) == 1:
matching_sub = subs_in_archive[0]
else:
for sub_name in subs_in_archive:
guess = guessit(sub_name)
# consider subtitle valid if:
# - episode and season match
# - format matches (if it was matched before)
# - release group matches (and we asked for one and it was matched, or it was not matched)
if guess["episode"] == subtitle.episode and guess[
"season"] == subtitle.season:
format_matches = "format" not in subtitle.matches or \
("format" in subtitle.matches and guess["format"].lower() in
subtitle.releases.lower())
release_group_matches = True
if subtitle.asked_for_release_group:
release_group_matches = "release_group" not in subtitle.matches or \
("release_group" in subtitle.matches and
guess["release_group"].lower() ==
subtitle.asked_for_release_group.lower())
if release_group_matches and format_matches:
matching_sub = sub_name
break
if not matching_sub:
raise ProviderError("None of expected subtitle found in archive")
subtitle.content = fix_line_ending(archive.read(matching_sub))
def download_subtitle(self, subtitle):
logger.info("Download subtitle %r", subtitle.link)
r = self.session.get(subtitle.link, params=None, timeout=10)
r.raise_for_status()
# check if it's rar or zip
if r.content[:4] == "Rar!":
compressed = RarFile(StringIO(r.content))
content = compressed.read(compressed.namelist()[0])
elif r.content[:4] == "PK":
compressed = ZipFile(StringIO(r.content))
content = compressed.read(compressed.namelist()[0])
else:
content = r.content
# some ppl just put wrong file inside
if mimetypes.guess_type(content)[0] != "text/plain":
raise ProviderError("Wrong subtitle inside compressed file")
subtitle.content = fix_line_ending(content)
def scan_archive(path):
"""Scan an archive from a `path`.
:param str path: existing path to the archive.
:return: the scanned video.
:rtype: :class:`~subliminal.video.Video`
"""
# check for non-existing path
if not os.path.exists(path):
raise ValueError('Path does not exist')
# check video extension
if not path.endswith(ARCHIVE_EXTENSIONS):
raise ValueError('%r is not a valid archive extension' %
os.path.splitext(path)[1])
dirpath, filename = os.path.split(path)
logger.info('Scanning archive %r in %r', filename, dirpath)
# rar extension
if filename.endswith('.rar'):
rar = RarFile(path)
# filter on video extensions
rar_filenames = [
f for f in rar.namelist() if f.endswith(VIDEO_EXTENSIONS)
]
# no video found
if not rar_filenames:
raise ValueError('No video in archive')
# more than one video found
if len(rar_filenames) > 1:
raise ValueError('More than one video in archive')
# guess
rar_filename = rar_filenames[0]
rar_filepath = os.path.join(dirpath, rar_filename)
video = Video.fromguess(rar_filepath, guessit(rar_filepath))
# size
video.size = rar.getinfo(rar_filename).file_size
else:
raise ValueError('Unsupported extension %r' %
os.path.splitext(path)[1])
return video
def run(self):
for idx, file in enumerate(self.input()):
ext = os.path.splitext(file.path)[-1]
if ext == '.rar':
arch = RarFile(file.path)
elif ext == '.zip':
arch = zipfile.ZipFile(file.path)
f = arch.namelist()[0]
target = os.path.join(os.getenv('TEMP_DIR'), f)
arch.extract(f, os.path.join(os.getenv('TEMP_DIR')))
copyfile(
target,
os.path.join(os.getenv('TEMP_DIR'),
os.path.basename(self.output()[idx].path)))
def __extract_cbr(self, file):
""" extract first image in cbr file, cache it in local cache folder"""
try:
archive_rar = RarFile(file, 'r')
extract_path = self.cache + path.basename(file)
if self.mode is 'all':
archive_rar.extractall(path=extract_path)
else:
first_file = archive_rar.namelist()[0]
archive_rar.extract(member=first_file, path=extract_path)
except BadRarFile as e:
raise e
finally:
archive_rar.close()
return extract_path
def run(self):
ext = os.path.splitext(self.input().path)[-1]
if ext == '.rar':
arch = RarFile(self.input().path)
elif ext == '.zip':
arch = zipfile.ZipFile(self.input().path)
for f in arch.namelist():
if os.path.basename(f) in [
os.path.basename(out.path) for out in self.output()
]:
target = os.path.join(os.getenv('TEMP_DIR'), f)
arch.extract(f, os.path.join(os.getenv('TEMP_DIR')))
copyfile(
target,
os.path.join(os.getenv('TEMP_DIR'), os.path.basename(f)))
def __init__(self, studentnr, specs, source_filename):
Submission.__init__(self, studentnr)
try:
source_file = RarFile(source_filename)
for spec in specs:
part = SubmissionPart(spec)
self.parts.append(part)
for filename in source_file.namelist():
filename = filename.replace('\\', '/')
info = source_file.getinfo(filename)
if not info.isdir() and spec.matches(filename):
part.add_data(filename, source_file.open(filename).read())
except BadRarFile:
pass
except NotRarFile:
pass
def extract_rar(complete_folder, rar_file, rar_name, data_name):
"""
Extract rar.gz file and get the data.
"""
rar_file_path = os.path.join(complete_folder, rar_file)
rf = RarFile(rar_file_path)
rar_inside = rf.namelist()
for t_i in rar_inside:
if rar_name in t_i:
rf.extract(t_i, path=complete_folder)
shutil.move(os.path.join(complete_folder, t_i), os.path.join(complete_folder, data_name))
break
try:
shutil.rmtree(os.path.join(complete_folder, t_i.split('/')[0]))
except:
pass
def scan_archive(path):
"""Scan an archive from a `path`.
:param str path: existing path to the archive.
:return: the scanned video.
:rtype: :class:`~subliminal.video.Video`
"""
# check for non-existing path
if not os.path.exists(path):
raise ValueError('Path does not exist')
# check video extension
if not path.endswith(ARCHIVE_EXTENSIONS):
raise ValueError('%r is not a valid archive extension' % os.path.splitext(path)[1])
dirpath, filename = os.path.split(path)
logger.info('Scanning archive %r in %r', filename, dirpath)
# rar extension
if filename.endswith('.rar'):
rar = RarFile(path)
# filter on video extensions
rar_filenames = [f for f in rar.namelist() if f.endswith(VIDEO_EXTENSIONS)]
# no video found
if not rar_filenames:
raise ValueError('No video in archive')
# more than one video found
if len(rar_filenames) > 1:
raise ValueError('More than one video in archive')
# guess
rar_filename = rar_filenames[0]
rar_filepath = os.path.join(dirpath, rar_filename)
video = Video.fromguess(rar_filepath, guessit(rar_filepath))
# size
video.size = rar.getinfo(rar_filename).file_size
else:
raise ValueError('Unsupported extension %r' % os.path.splitext(path)[1])
return video
async def fetch_file(session, url):
print(url)
async with session.get(url, timeout=60 * 60) as response:
if response.status != 200:
return await asyncio.sleep(0)
file_content = await response.content.read()
zipped_package = RarFile(BytesIO(file_content))
xml_filename = [
name for name in zipped_package.namelist() if name.endswith('.xml')
][0]
xml_file = zipped_package.read(xml_filename)
with open(os.path.join(ROOT, 'compendium', xml_filename),
'wb') as handler:
handler.write(xml_file)
return await response.text()
class ArchivedFile(object):
def __init__(self,filename):
if is_zipfile(filename):
self._zfile = ZipFile(filename)
elif is_rarfile(filename):
self._zfile = RarFile(filename)
def filelist(self):
return self._zfile.namelist()
def extract(self,srcfile,targetpath):
"extract srcfile in archive to targetpath"
for fullfilename in self.filelist():
if srcfile in fullfilename:
fpath,fname = os.path.split(fullfilename)
self._zfile.extract(fname.encode('gbk'),targetpath+fname)
return True
return None
def extractall(self,targetpath):
self._zfile.extractall(targetpath)
def unrar(path, rar_files, force, result): # pylint: disable=too-many-branches,too-many-statements
"""
Extracts RAR files
:param path: Path to look for files in
:param rar_files: Names of RAR files
:param force: process currently processing items
:param result: Previous results
:return: List of unpacked file names
"""
unpacked_dirs = []
if sickbeard.UNPACK == 1 and rar_files:
result.output += log_helper("Packed Releases detected: {0}".format(rar_files), logger.DEBUG)
for archive in rar_files:
failure = None
rar_handle = None
try:
archive_path = ek(os.path.join, path, archive)
if already_processed(path, archive, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(archive_path), logger.DEBUG)
continue
if not helpers.is_rar_file(archive_path):
continue
result.output += log_helper("Checking if archive is valid and contains a video: {0}".format(archive_path), logger.DEBUG)
rar_handle = RarFile(archive_path)
if rar_handle.needs_password():
# TODO: Add support in settings for a list of passwords to try here with rar_handle.set_password(x)
result.output += log_helper('Archive needs a password, skipping: {0}'.format(archive_path))
continue
# rar_handle.testrar()
# If there are no video files in the rar, don't extract it
rar_media_files = filter(helpers.is_media_file, rar_handle.namelist())
if not rar_media_files:
continue
rar_release_name = archive.rpartition('.')[0]
# Choose the directory we'll unpack to:
if sickbeard.UNPACK_DIR and os.path.isdir(sickbeard.UNPACK_DIR): # verify the unpack dir exists
unpack_base_dir = sickbeard.UNPACK_DIR
else:
unpack_base_dir = path
if sickbeard.UNPACK_DIR: # Let user know if we can't unpack there
result.output += log_helper('Unpack directory cannot be verified. Using {0}'.format(path), logger.DEBUG)
# Fix up the list for checking if already processed
rar_media_files = [os.path.join(unpack_base_dir, rar_release_name, rar_media_file) for rar_media_file in rar_media_files]
skip_rar = False
for rar_media_file in rar_media_files:
check_path, check_file = os.path.split(rar_media_file)
if already_processed(check_path, check_file, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(rar_media_file), logger.DEBUG)
skip_rar = True
break
if skip_rar:
continue
rar_extract_path = ek(os.path.join, unpack_base_dir, rar_release_name)
result.output += log_helper("Unpacking archive: {0}".format(archive), logger.DEBUG)
rar_handle.extractall(path=rar_extract_path)
unpacked_dirs.append(rar_extract_path)
except RarCRCError:
failure = ('Archive Broken', 'Unpacking failed because of a CRC error')
except RarWrongPassword:
failure = ('Incorrect RAR Password', 'Unpacking failed because of an Incorrect Rar Password')
except PasswordRequired:
failure = ('Rar is password protected', 'Unpacking failed because it needs a password')
except RarOpenError:
failure = ('Rar Open Error, check the parent folder and destination file permissions.',
'Unpacking failed with a File Open Error (file permissions?)')
except RarExecError:
failure = ('Invalid Rar Archive Usage', 'Unpacking Failed with Invalid Rar Archive Usage. Is unrar installed and on the system PATH?')
except BadRarFile:
failure = ('Invalid Rar Archive', 'Unpacking Failed with an Invalid Rar Archive Error')
except NeedFirstVolume:
continue
except (Exception, Error) as e:
failure = (ex(e), 'Unpacking failed')
finally:
if rar_handle:
del rar_handle
if failure:
result.output += log_helper('Failed to extract the archive {0}: {1}'.format(archive, failure[0]), logger.WARNING)
result.missed_files.append('{0} : Unpacking failed: {1}'.format(archive, failure[1]))
result.result = False
continue
return unpacked_dirs
from rarfile import RarFile
from rarfile import BadRarFile
from rarfile import PasswordRequired
import os
import threading
lock=threading.RLock()
rar=RarFile("/home/ljd/1.rar")
filename=rar.namelist()[0]
def loopkey(start,stop):
for x in xrange(start,stop):
try:
rar.setpassword(str(x))
rar.read(filename)
print x
os._exit(0)
except BadRarFile,PasswordRequired:
pass
def looprange(n):
data=range(0,n+1,n/100)
return [(value,data[index+1]) for index,value in enumerate(data[:-1])]
data=looprange(1000000000)
print data
for x,y in data:
threading.Thread(target=loopkey,args=(x,y)).start()
def handle(self, *args, **options):
update_flag = True
try:
latest_update = FiasUpdateInfo.objects.latest("create_ts")
except FiasUpdateInfo.DoesNotExist:
latest_update = None
fias_soap = Client(settings.FIAS_URL)
latest_soap = fias_soap.service.GetLastDownloadFileInfo()
version = latest_soap.VersionId
if latest_update:
if int(version) <= int(latest_update.version):
update_flag = False
if update_flag:
xurl = latest_soap.FiasDeltaXmlUrl
delta_file = urllib2.urlopen(xurl)
input_file = StringIO(delta_file.read())
new_update = FiasUpdateInfo(version=version)
new_update.textversion = latest_soap.TextVersion.encode("utf8")
new_update.delta_url = latest_soap.FiasDeltaXmlUrl
new_update.delta_file.save("fias_update_%s.rar" % version, ContentFile(input_file.getvalue()), save=False)
new_update.save()
# unpack, get xml, write to DB
rar_file = RarFile(new_update.delta_file.path)
update_file_addr = None
update_file_house = None
for packed_file in rar_file.namelist():
if packed_file.find("_ADDROBJ_") >= 0:
update_file_addr = packed_file
if packed_file.find("_HOUSE_") >= 0:
update_file_house = packed_file
# AddressObj
if not update_file_addr:
xml_string_addr = rar_file.read(update_file_addr)
xml_tree_addr = etree.fromstring(xml_string_addr)
update_items_addr = xml_tree_addr.getchildren()
if update_items_addr and update_items_addr[0].keys():
fields_addr = update_items_addr[0].keys()
update_length_addr = len(update_items_addr)
for counter_addr, update_item_addr in enumerate(update_items_addr):
new_addrobj = AddressObj()
for field_addr in fields_addr:
setattr(new_addrobj, field_addr, update_item_addr.get(field_addr))
new_addrobj.save()
print u"%s Address objects left" % (update_length_addr - counter_addr)
else:
print u"Wrong format of Address update file"
else:
print u"AddressObj file not found in the update"
# House
if update_file_house:
xml_string_house = rar_file.read(update_file_house)
xml_tree_house = etree.fromstring(xml_string_house)
update_items_house = xml_tree_house.getchildren()
if update_items_house and update_items_house[0].keys():
fields_house = update_items_house[0].keys()
update_length_house = len(update_items_house)
for counter_house, update_item_house in enumerate(update_items_house):
new_house = House()
for field_house in fields_house:
setattr(new_house, field_house, update_item_house.get(field_house))
new_house.save()
print u"%s House objects left" % (update_length_house - counter_house)
else:
print u"Wrong format of House update file"
else:
print u"House file not found in the update"
print u"Updated successfully"
else:
print u"No new updates found"
def downloader(code, key):
newDir()
error = exist = 0
isError = isExist = ''
nameList = listdir('PPT')
extensions = []
print('读取成功！开始下载')
pools = Semaphore(4)
threads = []
def down(e):
with pools:
downFilename = key + code[e]
if judgeFile(downFilename, nameList) == -1: return 0
try:
downloadUrl = processor(code[e])
extensions.append(downloadUrl[len(downloadUrl) -
4:len(downloadUrl)])
with open('temp\\{}.temp'.format(downFilename), 'wb') as f:
f.write(_get(downloadUrl, verify=False).content)
except:
pass
for each in range(len(code)):
threads.append(Thread(target=down, args=(each, )))
for each in threads:
each.start()
for each in range(len(threads)):
showBar(each + 1, len(code) + 1)
threads[each].join()
print('\n下载完毕！正在解压')
for each in range(len(code)):
showBar(each + 1, len(code) + 1)
filename = key + code[each]
judgeCode = judgeFile(filename, nameList, False)
if judgeCode == -1:
exist = exist + 1
continue
extension = extensions[each]
try:
move('temp\\{}.temp'.format(filename),
'PPT\\{}{}'.format(filename, extension))
if extension == '.rar':
package = RarFile('PPT\\{}.rar'.format(filename))
elif extension == '.zip':
package = ZipFile('PPT\\{}.zip'.format(filename))
else:
print('\r{}{}解压失败，格式不支持。{}'.format(filename, extension,
' ' * 5))
error = error + 1
continue
name = package.namelist()[0]
package.extractall('PPT')
package.close()
try:
rename('PPT\\{}'.format(name),
'PPT\\{}{}'.format(filename, name))
except:
pass
except:
print('\r{}{}解压失败，找不到文件。{}'.format(filename, extension, ' ' * 5))
error = error + 1
print('\n下载成功！处理文件中。。。')
for each in listdir('PPT'):
if path.splitext(each)[1] != '.ppt' and path.splitext(
each)[1] != '.pptx' and not path.isdir('PPT\\' + each):
remove('PPT\\' + each)
if error > 0: isError = '，{}个错误'.format(error)
if exist > 0: isExist = '，{}个文件已存在。'.format(exist)
print('PPT下载成功，共{}个{}{}'.format(len(code), isError, isExist))
while True:
pass
def add(self, archive_password=None, audio_file=None, session=None, artist_name_fallback=None):
cache_key = LibraryUpload.CACHE_KEY % (cherrypy.request.user.id, session)
all_tracks = None
if not cache.has(cache_key):
raise cherrypy.HTTPError(status=409)
else:
all_tracks = cache.get(cache_key)
if audio_file is not None and len(audio_file) == 0:
audio_file = None
if archive_password is not None and len(archive_password) == 0:
archive_password = None
content_disposition = cherrypy.request.headers.get('content-disposition')
filename = content_disposition[content_disposition.index('filename=') + 9:]
if filename.startswith('"') and filename.endswith('"'):
filename = filename[1:-1]
filename = unquote(filename)
ext = os.path.splitext(filename)[1].lower()[1:]
basename = os.path.splitext(filename)[0]
cache_path = os.path.join(cherrypy.config['opmuse'].get('cache.path'), 'upload')
if not os.path.exists(cache_path):
os.mkdir(cache_path)
tempdir = tempfile.mkdtemp(dir=cache_path)
path = os.path.join(tempdir, filename)
paths = []
rarfile.PATH_SEP = '/'
messages = []
with open(path, 'wb') as fileobj:
fileobj.write(cherrypy.request.rfile.read())
# this file is a regular file that belongs to an audio_file
if audio_file is not None:
track = None
tries = 0
# try and sleep until we get the track.. this will almost always
# be needed because of the async upload.
while track is None and tries < 10:
track = library_dao.get_track_by_filename(audio_file.encode('utf8'))
tries += 1
if track is None:
time.sleep(3)
if track is None:
messages.append(('warning', ("<strong>%s</strong>: Skipping <strong>%s</strong>, timeout trying to " +
"find its track.") % (audio_file, filename)))
else:
track_structure = TrackStructureParser(track)
track_path = track_structure.get_path(absolute=True)
relative_track_path = track_structure.get_path(absolute=False).decode('utf8', 'replace')
new_path = os.path.join(track_path, filename.encode('utf8'))
if os.path.exists(new_path):
messages.append(('warning', ("<strong>%s</strong>: Skipping <strong>%s</strong>, already exists " +
"in <strong>%s</strong>.") % (audio_file, filename, relative_track_path)))
else:
shutil.move(path.encode('utf8'), new_path)
messages.append(('info', ("<strong>%s</strong>: Uploaded <strong>%s</strong> to " +
"<strong>%s</strong>.") % (audio_file, filename, relative_track_path)))
elif ext == "zip":
# set artist name fallback to zip's name so if it's missing artist tags
# it's easily distinguishable and editable so it can be fixed after upload.
artist_name_fallback = basename
try:
zip = ZipFile(path)
if archive_password is not None:
zip.setpassword(archive_password.encode())
zip.extractall(tempdir)
os.remove(path)
for name in zip.namelist():
namepath = os.path.join(tempdir, name)
# ignore hidden files, e.g. OSX archive weirdness and such
if name.startswith(".") or os.path.split(name)[0] == "__MACOSX":
shutil.rmtree(namepath)
continue
paths.append(namepath.encode('utf8'))
except Exception as error:
messages.append(('danger', "<strong>%s</strong>: %s" % (os.path.basename(path), error)))
elif ext == "rar":
# look at corresponding ext == zip comment...
artist_name_fallback = basename
try:
rar = RarFile(path)
if archive_password is None and rar.needs_password():
messages.append(('danger', "<strong>%s</strong>: Needs password but none provided." %
os.path.basename(path)))
else:
if archive_password is not None:
rar.setpassword(archive_password)
rar.extractall(tempdir)
os.remove(path)
for name in rar.namelist():
namepath = os.path.join(tempdir, name)
if name.startswith("."):
shutil.rmtree(namepath)
continue
paths.append(namepath.encode('utf8'))
except Exception as error:
messages.append(('danger', "<strong>%s</strong>: %s" % (os.path.basename(path), error)))
# this is a plain audio file
else:
paths.append(path.encode('utf8'))
for path in paths:
# update modified time to now, we don't want the time from the zip
# archive or whatever
os.utime(path, None)
if len(paths) > 0:
tracks, add_files_messages = library_dao.add_files(paths, move=True, remove_dirs=False,
artist_name_fallback=artist_name_fallback,
user=cherrypy.request.user)
messages += add_files_messages
else:
tracks = []
shutil.rmtree(tempdir)
for track in tracks:
all_tracks.append(track.id)
if track.album is not None:
remotes.update_album(track.album)
if track.artist is not None:
remotes.update_artist(track.artist)
remotes.update_track(track)
hierarchy = Library._produce_track_hierarchy(library_dao.get_tracks_by_ids(all_tracks))
return {'hierarchy': hierarchy, 'messages': messages}
class ParserArchive:
def __init__(self, binary, html=None):
self.binary = io.BytesIO(binary)
self.archive_file = self.__load_archive()
self.tables_info = []
self.failures = {}
def __load_archive(self):
try:
self.archive_file = ZipFile(self.binary, 'r')
except BadZipfile:
try:
self.archive_file = RarFile(self.binary, 'r')
except (BadRarFile, NotRarFile, io.UnsupportedOperation,
NeedFirstVolume):
message = str(sys.exc_info()[1])
raise ValueError(f'архив не может быть прочитан ({message})')
return self.archive_file
def get_tables_info(self):
for file_name in self.archive_file.namelist():
file_ext = self.__get_file_ext(file_name)
if file_ext:
try:
self.__append_tables_info(self.archive_file, file_name,
file_ext)
except TypeError:
message = str(sys.exc_info()[1])
self.failures[file_name] = message
continue
return self.tables_info
def __append_tables_info(self, archive_file, inner_file_name,
inner_file_ext):
try:
inner_file = archive_file.read(inner_file_name)
binary = io.BytesIO(inner_file).read()
except (BadZipfile, BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError(f'{inner_file_ext}-архив не может быть прочитан')
if inner_file_ext == 'zip':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
zip_file = ZipFile(file_read, 'r')
for file_name in zip_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(zip_file, file_name, file_ext)
except BadZipfile:
raise TypeError('zip-архив не может быть прочитан')
elif inner_file_ext == 'rar':
file = archive_file.read(inner_file_name)
try:
file_read = io.BytesIO(file)
rar_file = RarFile(file_read)
for file_name in rar_file.namelist():
file_ext = self.__get_file_ext(file_name)
self.__append_tables_info(rar_file, file_name, file_ext)
except (BadRarFile, NotRarFile, io.UnsupportedOperation):
raise TypeError('rar-архив не может быть прочитан')
parser = self.__choose_parser(inner_file_ext)(binary=binary, html=None)
tables_info = parser.get_tables_info()
for ti in tables_info:
self.tables_info.append(ti)
def __get_file_ext(self, full_file_name):
full_file_name = full_file_name.lower()
splitted = full_file_name.split('.')
file_ext = None
if len(
splitted
) > 1: # в противном случае наткнулись на название директории, а не файла
file_ext = splitted[-1]
return file_ext
def __choose_parser(self, file_ext):
parsers_by_ext = {
'xlsx': ParserXLSX,
'xls': ParserXLS,
'docx': ParserDOCX,
'zip': ParserArchive,
'rar': ParserArchive,
'htm': ParserHTM,
}
if file_ext in parsers_by_ext:
return parsers_by_ext[file_ext]
else:
raise TypeError(
f'подходящий парсер не найден (расширение {file_ext})')
def unrar( filename, destination=None, report=False ):
from rarfile import RarFile
base_dir = ""
if destination is None:
destination = os.path.dirname( filename )
try:
rar = RarFile( filename, "r" )
namelist = rar.namelist()
total_items = len( namelist ) or 1
diff = 100.0 / total_items
percent = 0
# nom du dossier racine
root_dir = namelist[ -1 ]
is_root_dir = True
# si root_dir n'est pas un dossier ou n'est pas la racine, on se base sur le nom de l'archive
#print root_dir
if not rar.getinfo( root_dir ).isdir():
is_root_dir = False
else:
for i in namelist:
#print root_dir in i, i
if not root_dir in i:
is_root_dir = False
break
if not is_root_dir:#rar.getinfo( root_dir ).isdir():
root_dir = os.path.basename( os.path.splitext( filename )[ 0 ] )
base_dir = os.path.join( destination, root_dir )
if os.path.isdir( base_dir ):
shutil2.rmtree( base_dir )
os.makedirs( base_dir )
time_sleep = get_time_sleep( filename )
# avec cette methode on extract dans le dossier ou est l'archive
ok = executebuiltin( 'XBMC.Extract(%s)' % ( filename, ) )
#sleep une seconde ou plus selon la drosseur du fichier le temps que le builtin est fini car l'action suivante
# "os.listdir" est excecuter avant la fin du builtin
sleep( time_sleep )
# si le dossier base_dir est vide on move les items de namelist dedans
if not os.listdir( base_dir ):
for item in namelist:
src = os.path.normpath( os.path.join( os.path.dirname( filename ), item ) )
dst = os.path.normpath( os.path.join( base_dir, item ) )
if not rar.getinfo( item ).isdir():
if not os.path.isdir( os.path.dirname( dst ) ):
os.makedirs( os.path.dirname( dst ) )
shutil2.move( src, dst, overwrite=True )
elif os.path.exists( src ) and not os.listdir( src ):
shutil2.rmtree( src )
#maintenant on verifier l'extraction d'xbmc avec la liste de la lib rarfile
if os.path.isdir( base_dir ):
size = 0
list_size = 0
if not root_dir in namelist:
list_size -= 1
namelist = [ os.path.split( item )[ 1 ] for item in namelist ]
for root, dirs, files in os.walk( base_dir, topdown=False ):
percent += diff
list_size += 1
for file in files:
percent += diff
list_size += 1
if report:
if DIALOG_PROGRESS.iscanceled():
break
DIALOG_PROGRESS.update( int( percent ), Language( 187 ) % ( list_size, total_items ), file, Language( 110 ) )
#print round( percent, 2 ), file
if file in namelist:
size += os.path.getsize( os.path.join( root, file ) )
else:
print "Error %s est dans la liste de depart!" % file
#print size
if not size:
print "Error for extracting rar: %s" % filename
rar.close()
del rar
# si list_size est pas declarer une erreur automatique est creer ;)
return base_dir, list_size == total_items
except:
print_exc()
return "", False
def unrar(filename, destination=None, report=False):
from rarfile import RarFile
base_dir = ""
if destination is None:
destination = os.path.dirname(filename)
try:
rar = RarFile(filename, "r")
namelist = rar.namelist()
total_items = len(namelist) or 1
diff = 100.0 / total_items
percent = 0
# nom du dossier racine
root_dir = namelist[-1]
is_root_dir = True
# si root_dir n'est pas un dossier ou n'est pas la racine, on se base sur le nom de l'archive
if not rar.getinfo(root_dir).isdir():
is_root_dir = False
else:
for i in namelist:
#print root_dir in i, i
if not root_dir in i:
is_root_dir = False
break
if not is_root_dir: #rar.getinfo( root_dir ).isdir():
root_dir = os.path.basename(os.path.splitext(filename)[0])
base_dir = os.path.join(destination, root_dir)
if os.path.isdir(base_dir):
shutil2.rmtree(base_dir)
os.makedirs(base_dir)
time_sleep = get_time_sleep(filename)
# avec cette methode on extract dans le dossier ou est l'archive
ok = executebuiltin('XBMC.Extract(%s)' % (filename, ))
#sleep une seconde ou plus selon la drosseur du fichier le temps que le builtin est fini car l'action suivante
# "os.listdir" est excecuter avant la fin du builtin
sleep(time_sleep)
# si le dossier base_dir est vide on move les items de namelist dedans
if not os.listdir(base_dir):
for item in namelist:
src = os.path.normpath(
os.path.join(os.path.dirname(filename), item))
dst = os.path.normpath(os.path.join(base_dir, item))
if not rar.getinfo(item).isdir():
if not os.path.isdir(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
shutil2.move(src, dst, overwrite=True)
elif os.path.exists(src) and not os.listdir(src):
shutil2.rmtree(src)
#maintenant on verifier l'extraction d'xbmc avec la liste de la lib rarfile
if os.path.isdir(base_dir):
size = 0
list_size = 0
if not root_dir in namelist:
list_size -= 1
namelist = [os.path.split(item)[1] for item in namelist]
for root, dirs, files in os.walk(base_dir, topdown=False):
percent += diff
list_size += 1
for file in files:
percent += diff
list_size += 1
if report:
if DIALOG_PROGRESS.iscanceled():
break
DIALOG_PROGRESS.update(
int(percent),
_(30187) % (list_size, total_items), file,
_(30110))
#print round( percent, 2 ), file
if file in namelist:
size += os.path.getsize(os.path.join(root, file))
else:
print "Error %s est dans la liste de depart!" % file
#print size
if not size:
print "Error for extracting rar: %s" % filename
rar.close()
del rar
# si list_size est pas declarer une erreur automatique est creer ;)
return base_dir, list_size == total_items
except:
print_exc()
return "", False
from rarfile import RarFile
path = '/Users/username/Downloads/'
filename = 'py_20200326_174202.rar'
filepath = 'temp/'
rf = RarFile(path + filename, mode='r') # mode的值只能为'r'
rf_list = rf.namelist() # 得到压缩包里所有的文件
print('rar文件内容', rf_list)
for f in rf_list:
rf.extract(f, path + filepath) # 循环解压，将文件解压到指定路径
# 一次性解压所有文件到指定目录
# rf.extractall(path) # 不传path，默认为当前目录
def unrar(path, rar_files, force, result): # pylint: disable=too-many-branches,too-many-statements
"""
Extracts RAR files
:param path: Path to look for files in
:param rar_files: Names of RAR files
:param force: process currently processing items
:param result: Previous results
:return: List of unpacked file names
"""
unpacked_dirs = []
if sickbeard.UNPACK == 1 and rar_files:
result.output += log_helper("Packed Releases detected: {0}".format(rar_files), logger.DEBUG)
for archive in rar_files:
failure = None
rar_handle = None
try:
archive_path = ek(os.path.join, path, archive)
if already_processed(path, archive, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(archive_path), logger.DEBUG)
continue
if not helpers.is_rar_file(archive_path):
continue
result.output += log_helper("Checking if archive is valid and contains a video: {0}".format(archive_path), logger.DEBUG)
rar_handle = RarFile(archive_path)
if rar_handle.needs_password():
# TODO: Add support in settings for a list of passwords to try here with rar_handle.set_password(x)
result.output += log_helper('Archive needs a password, skipping: {0}'.format(archive_path))
continue
# rar_handle.testrar()
# If there are no video files in the rar, don't extract it
rar_media_files = filter(helpers.is_media_file, rar_handle.namelist())
if not rar_media_files:
continue
rar_release_name = archive.rpartition('.')[0]
# Choose the directory we'll unpack to:
if sickbeard.UNPACK_DIR and os.path.isdir(sickbeard.UNPACK_DIR): # verify the unpack dir exists
unpack_base_dir = sickbeard.UNPACK_DIR
else:
unpack_base_dir = path
if sickbeard.UNPACK_DIR: # Let user know if we can't unpack there
result.output += log_helper('Unpack directory cannot be verified. Using {0}'.format(path), logger.DEBUG)
# Fix up the list for checking if already processed
rar_media_files = [os.path.join(unpack_base_dir, rar_release_name, rar_media_file) for rar_media_file in rar_media_files]
skip_rar = False
for rar_media_file in rar_media_files:
check_path, check_file = os.path.split(rar_media_file)
if already_processed(check_path, check_file, force, result):
result.output += log_helper(
"Archive file already post-processed, extraction skipped: {0}".format
(rar_media_file), logger.DEBUG)
skip_rar = True
break
if skip_rar:
continue
rar_extract_path = ek(os.path.join, unpack_base_dir, rar_release_name)
result.output += log_helper("Unpacking archive: {0}".format(archive), logger.DEBUG)
rar_handle.extractall(path=rar_extract_path)
unpacked_dirs.append(rar_extract_path)
except RarCRCError:
failure = ('Archive Broken', 'Unpacking failed because of a CRC error')
except RarWrongPassword:
failure = ('Incorrect RAR Password', 'Unpacking failed because of an Incorrect Rar Password')
except PasswordRequired:
failure = ('Rar is password protected', 'Unpacking failed because it needs a password')
except RarOpenError:
failure = ('Rar Open Error, check the parent folder and destination file permissions.',
'Unpacking failed with a File Open Error (file permissions?)')
except RarExecError:
failure = ('Invalid Rar Archive Usage', 'Unpacking Failed with Invalid Rar Archive Usage. Is unrar installed and on the system PATH?')
except BadRarFile:
failure = ('Invalid Rar Archive', 'Unpacking Failed with an Invalid Rar Archive Error')
except NeedFirstVolume:
continue
except (Exception, Error) as e:
failure = (ex(e), 'Unpacking failed')
finally:
if rar_handle:
del rar_handle
if failure:
result.output += log_helper('Failed to extract the archive {0}: {1}'.format(archive, failure[0]), logger.WARNING)
result.missed_files.append('{0} : Unpacking failed: {1}'.format(archive, failure[1]))
result.result = False
continue
return unpacked_dirs
def raro_flist(apath):
raro = RarFile(apath)
return raro, raro.namelist()
