def content(request=None):
base = Path(current_app.config.get('INUPYPI_REPO', Path('.', 'packages')))
if request:
repo = Path(base, request)
else:
repo = base
try:
repo = repo.absolute()
base = base.absolute()
if not repo.exists():
if base == repo:
raise InuPyPIMissingRepoPath
# sets the request to lowercase and compares it with
# the existing items in the repository in lowercase
repo = search_path(repo, base)
if not repo:
raise InuPyPI404Exception
if repo.isdir():
return Dirs(repo)
if repo.isfile():
return repo
except InuPyPIMissingRepoPath:
abort(500, 'Missing repository or package path!')
except InuPyPI404Exception:
abort(404, 'Path or File could not be found!')
except:
abort(500, 'Internal Server Error!')
return repo
def test_email_schedule():
"""Tests that the schedule update email sends properly."""
emails = ['*****@*****.**', '*****@*****.**']
template_text = Path('tests/files/update.txt')
template_html = Path('tests/files/update.html')
custom_config = deepcopy(APP_CONFIG)
custom_config['email']['update_text'] = template_text.absolute()
custom_config['email']['update_html'] = template_html.absolute()
notification_details = {
'additions': [],
'deletions': [],
'changes': [],
'missing': [],
'null': [],
}
# Will need to update exceptions if any occur in production
try:
notify.email_schedule(USER, emails, custom_config,
notification_details)
except OSError:
assert False
else:
assert True
def handle(self, *args, **options):
from ....media.models import MediaFile
from ...models import Transcript
if len(args) != 1:
raise CommandError('Provide media URL.')
(url,) = args
local_path = Path(url)
if local_path.exists():
url = 'file://{}'.format(local_path.absolute())
media_file = MediaFile.objects.create(
data_url=url,
)
if options['verbosity']:
self.stdout.write('Created media file: {}'.format(media_file))
transcript = Transcript.objects.create(
name=url,
)
if options['verbosity']:
self.stdout.write('Created transcript: {}'.format(transcript))
def create_file(self, name, contents):
"""
Creates a gzip file
:param name: (str) name of the file to be created
:param contents: (str) contents to be written in the file
:return: (str or False) path of the created file
"""
# write a tmp file
tmp = mkstemp()[1]
with gzip.open(tmp, 'wb') as handler:
handler.write(contents)
# send it to the FTP server
if self.ftp:
self.ftp.storbinary('STOR {}'.format(name), open(tmp, 'rb'))
return '{}{}'.format(self.path, name)
# or save it locally
else:
new_path = Path(self.path).child(name)
tmp_path = Path(tmp)
tmp_path.copy(new_path)
if new_path.exists():
return new_path.absolute()
return False
def create_file(self, name, contents):
"""
Creates a gzip file
:param name: (str) name of the file to be created
:param contents: (str) contents to be written in the file
:return: (str or False) path of the created file
"""
# write a tmp file
tmp = mkstemp()[1]
with gzip.open(tmp, 'wb') as handler:
handler.write(contents)
# send it to the FTP server
if self.ftp:
self.ftp.storbinary('STOR {}'.format(name), open(tmp, 'rb'))
return '{}{}'.format(self.path, name)
# or save it locally
else:
new_path = Path(self.path).child(name)
tmp_path = Path(tmp)
tmp_path.copy(new_path)
if new_path.exists():
return new_path.absolute()
return False
class ReleaseUnpackerRarFile(object):
"""Release unpacker RAR file."""
def __init__(self, rar_file_path):
"""Initialize and validate rar file path."""
self.rar_file_path = Path(rar_file_path)
if (not self.rar_file_path.exists() or not self.rar_file_path.isfile()
or not self.rar_file_path.ext == ".rar"):
raise ReleaseUnpackerRarFileError("Invalid RAR file {}".format(
self.rar_file_path))
self.rar_file_path_abs = self.rar_file_path.absolute()
self.rar_file = rarfile.RarFile(self.rar_file_path)
def __repr__(self):
"""Return object string representation."""
return "<ReleaseUnpackerRarFile: {}>".format(self.rar_file_path_abs)
@lazy
def name(self):
"""Return name of release folder."""
if self.subs_dir:
name = self.rar_file_path.parent.parent.name
else:
name = self.rar_file_path.parent.name
return str(name)
@lazy
def subs_dir(self):
"""Return True if RAR file is located in a Subs folder."""
if self.rar_file_path.parent.name.lower() in ("subs", "sub"):
return True
else:
return False
@lazy
def file_list(self):
"""Return file list of RAR file."""
files = []
for file in self.rar_file.infolist():
files.append({"name": Path(file.filename), "size": file.file_size})
return files
def extract_file(self, file_name, unpack_dir):
"""Extract file_name and return extracted file path."""
self.rar_file.extract(file_name, path=unpack_dir)
self.extracted_file_path = Path(unpack_dir, file_name)
# Set the mtime to current time
self.set_mtime()
return self.extracted_file_path
def set_mtime(self):
"""Set mtime of extracted file path to current time."""
os.utime(self.extracted_file_path, None)
def test_email_missing_codes():
"""Tests that the missing codes email sends properly."""
template_text = Path('tests/files/missing_codes.txt')
template_html = Path('tests/files/missing_codes.html')
custom_config = deepcopy(APP_CONFIG)
custom_config['email']['missing_codes_text'] = template_text.absolute()
custom_config['email']['missing_codes_html'] = template_html.absolute()
missing_codes = ['A1', 'A2']
# Will need to update exceptions if any occur in production
try:
notify.email_missing_codes(missing_codes, custom_config)
except MessageError:
assert False
else:
assert True
def test_email_welcome():
"""Tests that the welcome email sends properly."""
emails = ['*****@*****.**', '*****@*****.**']
template_text = Path('tests/files/welcome.txt')
template_html = Path('tests/files/welcome.html')
custom_config = deepcopy(APP_CONFIG)
custom_config['email']['welcome_text'] = template_text.absolute()
custom_config['email']['welcome_html'] = template_html.absolute()
# Will need to update exceptions if any occur in production
try:
notify.email_welcome(USER, emails, custom_config)
except OSError:
assert False
else:
assert True
def auto(options):
'''
Load my task options
'''
this_file = Path(__file__)
cfg_file = this_file.absolute().parent.child('pavement_config.yml')
cfg = bunchify_yaml_file(cfg_file)
options.cfg = cfg
def auto(options):
'''
Load my task options
'''
this_file = Path(__file__)
cfg_file = this_file.absolute().parent.child('pavement_config.yml')
cfg = bunchify_yaml_file(cfg_file)
options.cfg = cfg
def unused_write_release_notes():
"""
Generate docs/releases/x.y.z.rst file from setup_info.
"""
v = env.SETUP_INFO['version']
if v.endswith('+'):
return
notes = Path(env.ROOTDIR, 'docs', 'releases', '%s.rst' % v)
if notes.exists():
return
must_confirm("Create %s" % notes.absolute())
#~ context = dict(date=get_current_date().strftime(env.long_date_format))
context = dict(date=get_current_date().strftime('%Y%m%d'))
context.update(env.SETUP_INFO)
txt = """\
==========================
Version %(version)s
==========================
Release process started :blogref:`%(date)s`
List of changes
===============
New features
------------
Optimizations
-------------
Bugfixes
--------
Manual tasks after upgrade
--------------------------
""" % context
notes.write_file(txt)
notes.parent.child('index.rst').set_times()
args = [os.environ['EDITOR']]
args += [notes.absolute()]
local(' '.join(args))
def test_load_directionals_no_space_end_of_line(self):
temp_dir = Path(tempfile.mkdtemp())
temp_file = Path(temp_dir, 'test.txt')
temp_file.write_file('Northeast NE\n')
Directional.load_directionals(temp_file.absolute())
temp_dir.rmtree()
self.assertEqual('Northeast', Directional.objects.first().direction)
self.assertEqual('NE', Directional.objects.first().abbreviation)
def test_load_street_types_no_space_end_of_line(self):
temp_dir = Path(tempfile.mkdtemp())
temp_file = Path(temp_dir, 'test.txt')
temp_file.write_file('VILLAGE VILL VLG\n')
StreetType.load_street_types(temp_file.absolute())
temp_dir.rmtree()
self.assertEqual('VILLAGE VILL', StreetType.objects.first().name)
self.assertEqual('VLG', StreetType.objects.first().abbreviation)
def __check_directory(self):
"""
Check if the entered directory exists
:return: (unipath.Path or False) the path to the existing directory
"""
directory = Path(self.arguments['<directory>'])
if not directory.exists() or not directory.isdir():
msg = '{} is not a valid directory'.format(directory.absolute())
self.__output(msg, error=True)
return False
return directory
def test_load_states_space_end_of_line(self):
temp_dir = Path(tempfile.mkdtemp())
temp_file = Path(temp_dir, 'test.txt')
temp_file.write_file('Florida FL \n')
State.load_states(temp_file.absolute())
temp_dir.rmtree()
self.assertEqual('Florida', State.objects.first().name)
self.assertEqual('FL', State.objects.first().abbreviation)
class ReleaseUnpackerRarFile(object):
def __init__(self, rar_file_path):
self.rar_file_path = Path(rar_file_path)
if (not self.rar_file_path.exists() or not self.rar_file_path.isfile()
or not self.rar_file_path.ext == '.rar'):
raise ReleaseUnpackerRarFileError('Invalid RAR file {}'.format(
self. rar_file_path))
self.rar_file_path_abs = self.rar_file_path.absolute()
self.rar_file = rarfile.RarFile(self.rar_file_path)
def __repr__(self):
return '<ReleaseUnpackerRarFile: {}>'.format(self.rar_file_path_abs)
@lazy
def name(self):
if self.subs_dir:
name = self.rar_file_path.parent.parent.name
else:
name = self.rar_file_path.parent.name
return str(name)
@lazy
def subs_dir(self):
if self.rar_file_path.parent.name.lower() in ('subs', 'sub'):
return True
else:
return False
@lazy
def file_list(self):
files = []
for file in self.rar_file.infolist():
files.append({'name': Path(file.filename), 'size': file.file_size})
return files
def set_mtime(self):
with file(self.extracted_file_path, 'a'):
os.utime(self.extracted_file_path, None)
def extract_file(self, file_name, unpack_dir):
self.rar_file.extract(file_name, path=unpack_dir)
self.extracted_file_path = Path(unpack_dir, file_name)
# Set the mtime to current time
self.set_mtime()
return self.extracted_file_path
def parse(path='./flat_files/'):
path = Path(path)
print "parsing records at {}".format(path.absolute())
records = []
for p in path.listdir():
try:
gbr = GenBank.read(open(p))
records.append(gbr)
except:
print 'error with file', p
print "parsed %s records.." % len(records)
return records
def parse(path='./flat_files/'):
path = Path(path)
print "parsing records at {}".format(path.absolute())
records = []
for p in path.listdir():
try:
gbr = GenBank.read(open(p))
records.append(gbr)
except:
print 'error with file', p
print "parsed %s records.." % len(records)
return records
def get_current_release_name():
run_args = {}
# Append capture value if we are running locally
if env.run.__name__ == "elocal":
run_args["capture"] = True
path = env.run(
"ls -dt %s/*/ | sort -n -t _ -k 2 | tail -1" % get_releases_path(),
**run_args)
if not path:
return
release = Path(path)
try:
int(release.absolute().name)
except ValueError as e:
print e
return
return release.absolute().name
def get_current_release_name():
run_args = {}
# Append capture value if we are running locally
if env.run.__name__ == "elocal":
run_args["capture"] = True
path = env.run(
"ls -dt %s/*/ | sort -n -t _ -k 2 | tail -1" %
get_releases_path(), **run_args)
if not path:
return
release = Path(path)
try:
int(release.absolute().name)
except ValueError as e:
print e
return
return release.absolute().name
def content(request=None):
base = Path(current_app.config.get('INUPYPI_REPO',
Path('.', 'packages')))
if request:
repo = Path(base, request)
else:
repo = base
try:
repo = repo.absolute()
base = base.absolute()
if not repo.exists():
if base == repo:
raise InuPyPIMissingRepoPath
# sets the request to lowercase and compares it with
# the existing items in the repository in lowercase
repo = search_path(repo, base)
if not repo:
raise InuPyPI404Exception
if repo.isdir():
return Dirs(repo)
if repo.isfile():
return repo
except InuPyPIMissingRepoPath:
abort(500, 'Missing repository or package path!')
except InuPyPI404Exception:
abort(404, 'Path or File could not be found!')
except:
abort(500, 'Internal Server Error!')
return repo
def yolo_detect_image_task(analysed_image_id):
from src.core.models import AnalysedImage
try:
db_image = AnalysedImage.objects.get(id=analysed_image_id)
except AnalysedImage.DoesNotExist:
return None
filename = db_image.image.name.split('/')[-1]
clean_filename = filename.split('.')[0]
temp_file = settings.TEMP_DIR.child(filename)
with open(temp_file, 'bw') as fd:
fd.write(db_image.image.read())
pred_file = Path(settings.DARKNET_DIR,
'pred-{}.png'.format(clean_filename))
command = ' '.join([
settings.DARKNET_BIN,
'detect',
settings.YOLO_CONF,
settings.YOLO_WEIGHTS,
temp_file,
'-out',
pred_file.name.split('.')[0],
])
print('Exec --> {}'.format(command))
detect = subprocess.Popen(
shlex.split(command),
#stdout=subprocess.DEVNULL,
#stderr=subprocess.DEVNULL,
cwd=settings.DARKNET_DIR,
)
detect.wait()
if not pred_file.exists():
pred_file = Path(settings.DARKNET_DIR,
'pred-{}.jpg'.format(clean_filename))
print("Finshed yolo analysis!")
print("Prediction file is located at: " + pred_file.absolute())
db_image.write_yolo_file(pred_file)
db_image.save()
temp_file.remove()
pred_file.remove()
print('Success!')
def download_zipped_corpus():
tempdir = tempfile.mkdtemp()
url = 'https://github.com/torvalds/linux/archive/master.zip'
test_url = 'https://github.com/facebook/libphenom/archive/master.zip'
file_name = url.split('/')[-1]
# download zipfile with output to console
def clear(): os.system('cls' if os.name=='nt' else 'clear')
wget_out_file = Path(tempdir, file_name)
wget = sub.Popen(['wget', url,'-O', wget_out_file], stdout=sub.PIPE, stderr=sub.STDOUT)
while True:
line = wget.stdout.readline()
if not line: break
clear()
print line
wget.wait()
return wget_out_file.absolute()
def handle(self, *args, **options):
from ....media.models import MediaFile
from ...models import Transcript
if len(args) != 1:
raise CommandError('Provide media URL.')
(url, ) = args
local_path = Path(url)
if local_path.exists():
url = 'file://{}'.format(local_path.absolute())
media_file = MediaFile.objects.create(data_url=url, )
if options['verbosity']:
self.stdout.write('Created media file: {}'.format(media_file))
transcript = Transcript.objects.create(name=url, )
if options['verbosity']:
self.stdout.write('Created transcript: {}'.format(transcript))
class ReleaseUnpacker(object):
"""ReleaseUnpacker."""
def __init__(self,
release_search_dir,
tmp_dir,
unpack_dir,
no_remove=False):
"""Initialize and validate ReleaseUnpacker."""
self.release_search_dir = Path(release_search_dir)
self.release_search_dir_abs = self.release_search_dir.absolute()
self.tmp_dir = Path(tmp_dir)
self.unpack_dir = Path(unpack_dir)
self.no_remove = no_remove
if not self.release_search_dir_abs.exists():
raise ReleaseUnpackerError(
"Release search dir {} doesn't exist".format(
self.release_search_dir))
elif not self.release_search_dir_abs.isdir():
raise ReleaseUnpackerError(
"Release search dir {} is not a dir".format(
self.release_search_dir))
elif not self.tmp_dir.exists():
raise ReleaseUnpackerError("Tmp dir {} doesn't exist".format(
self.tmp_dir))
elif not self.tmp_dir.isdir():
raise ReleaseUnpackerError("Tmp dir {} is not a dir".format(
self.tmp_dir))
elif not self.unpack_dir.exists():
raise ReleaseUnpackerError("Unpack dir {} doesn't exist".format(
self.unpack_dir))
elif not self.unpack_dir.isdir():
raise ReleaseUnpackerError("Unpack dir {} is not a dir".format(
self.unpack_dir))
def __repr__(self):
"""Return object string representation."""
return "<ReleaseUnpacker: {} ({}) ({})>".format(
self.release_search_dir_abs, self.tmp_dir, self.unpack_dir)
def file_exists_size_match(self, unpack_file_path, size_in_rar):
"""Return True if unpack_file_path exists already and size matches."""
if (unpack_file_path.exists()
and unpack_file_path.size() == size_in_rar):
log.info("%s already exists and size match", unpack_file_path)
return True
else:
return False
def unpack_release_dir_rars(self):
"""Run unpacker.
Unpack all whitelisted file extensions found in RAR files.
"""
# Scan for RAR files
self.rar_files = self.scan_rars()
if not self.rar_files:
log.debug("No RARs found in %s", self.release_search_dir_abs)
return False
# Process the RAR files in any were found
for rar_file_path in self.rar_files:
log.debug("Found RAR file %s", rar_file_path)
release_unpacker_rar_file = ReleaseUnpackerRarFile(rar_file_path)
if release_unpacker_rar_file.subs_dir:
self.unpack_subs_rar(release_unpacker_rar_file)
else:
self.unpack_rar(release_unpacker_rar_file)
# Remove release dirs when unpack is done
self.remove_release_dirs()
return self
def scan_rars(self):
"""Scan release_search_dir for .rar files.
Find all sub folders and return a list of the first .rar file in
each folder if any is found.
"""
scan_dirs = [
dir for dir in self.release_search_dir_abs.walk(filter=DIRS)
]
scan_dirs.append(self.release_search_dir_abs)
rar_files = []
for dir in scan_dirs:
rar_files_found = dir.listdir(pattern="*.rar", filter=FILES)
if rar_files_found:
rar_files.append(rar_files_found[0])
return rar_files
def remove_release_dirs(self):
"""Remove all release dirs from rar_files list."""
for rar_file_path in self.rar_files:
release_dir = rar_file_path.parent
if release_dir.exists():
if self.no_remove:
log.info("No remove active, not removing %s", release_dir)
else:
log.info("Unpack complete, removing %s", release_dir)
release_dir.rmtree()
def unpack_subs_rar(self, release_unpacker_rar_file):
"""Unpack a RAR in a Subs folder."""
log.debug(
"Processing subs RAR file %s",
release_unpacker_rar_file.rar_file_path_abs,
)
for rarfile_file in release_unpacker_rar_file.file_list:
# File in RAR is not a RAR file, extract
if rarfile_file["name"].ext != ".rar":
unpack_filename = "{}{}".format(release_unpacker_rar_file.name,
rarfile_file["name"].ext)
unpack_file_path_abs = Path(self.unpack_dir, unpack_filename)
# File exists and size match
if self.file_exists_size_match(unpack_file_path_abs,
rarfile_file["size"]):
continue
self.unpack_move_rar_file(
release_unpacker_rar_file,
rarfile_file["name"],
unpack_file_path_abs,
)
# RAR file in RAR, extract to Subs folder and extract RAR
else:
log.debug(
"RAR file %s in %s",
rarfile_file["name"],
release_unpacker_rar_file.rar_file_path_abs,
)
# Extract the RAR to Subs folder
subs_dir = release_unpacker_rar_file.rar_file_path_abs.parent
log.debug("Extracting %s to %s", rarfile_file["name"],
subs_dir)
extracted_file_path = release_unpacker_rar_file.extract_file(
rarfile_file["name"], subs_dir)
# Extract the extracted Subs RAR file
self.unpack_subs_rar(
ReleaseUnpackerRarFile(extracted_file_path))
# Remove RAR file in Subs folder
log.debug("Removing extracted Subs RAR %s",
extracted_file_path)
extracted_file_path.remove()
def unpack_rar(self, release_unpacker_rar_file):
"""Unpack RAR files. Only process whitelisted file extensions."""
for rarfile_file in release_unpacker_rar_file.file_list:
# Check file extension
if rarfile_file["name"].ext not in (
".avi",
".mkv",
".img",
".iso",
".mp4",
):
log.info("Skipping %s, unwanted ext", rarfile_file["name"])
continue
unpack_filename = "{}{}".format(release_unpacker_rar_file.name,
rarfile_file["name"].ext)
unpack_file_path_abs = Path(self.unpack_dir, unpack_filename)
# File exists and size match
if self.file_exists_size_match(unpack_file_path_abs,
rarfile_file["size"]):
continue
# Unpack file in RAR
self.unpack_move_rar_file(
release_unpacker_rar_file,
rarfile_file["name"],
unpack_file_path_abs,
)
return True
def unpack_move_rar_file(self, release_unpacker_rar_file,
rarfile_file_name, unpack_file_path):
"""Unpack and move RAR file.
Extract an individual file from release_unpacker_rar_file to
unpack_file_path.
"""
# Extract file to tmp_dir
log.debug("Extracting %s to %s", rarfile_file_name, self.tmp_dir)
log.info("%s unpack started", unpack_file_path.name)
unpack_start = datetime.now().replace(microsecond=0)
extracted_file_path = release_unpacker_rar_file.extract_file(
rarfile_file_name, self.tmp_dir)
unpack_end = datetime.now().replace(microsecond=0)
unpack_time = human(unpack_end - unpack_start, past_tense="{}")
if not unpack_time:
log.info("%s unpack done", unpack_file_path.name)
else:
log.info("%s unpack done, %s", unpack_file_path.name, unpack_time)
# Move file and rename to unpack_dir
log.debug("Moving %s to %s", extracted_file_path, unpack_file_path)
extracted_file_path.move(unpack_file_path)
import sys
import django
from unipath import Path
from django.conf import settings
from django.core.management import execute_from_command_line
BASE_DIR = Path(__file__).ancestor(2)
print(BASE_DIR)
sys.path.append(BASE_DIR.absolute())
settings.configure(DATABASES={
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR.child('db.sqlite3')
}
},
INSTALLED_APPS=[
'django_pg_commands',
],
PG_COMMAND_TOOLS=BASE_DIR.child('configuration.json'))
django.setup()
args = [sys.argv[0], 'test']
# Current module (``tests``) and its submodules.
test_cases = '.'
# # Allow accessing test options from the command line.
#! coding=utf-8
import datetime, smtplib, imp, os
from email.mime.text import MIMEText
import requests
from git import Repo
from git.errors import *
from unipath import Path
from jinja2 import Template
# Local settings are stored in $HOME/.ometrics/settings.py
# Find and load module there, creating files/dirs where appropriate
om = Path( os.environ['HOME'] ).absolute().child('.ometrics')
om.mkdir()
module_path = om.absolute()
if not om.child('settings.py').exists():
print('Copying setitngs.py into %s' % str(om.child('settings.py')))
Path(__file__).parent.child('settings.py').copy(om.child('settings.py'))
imp.load_source('settings', module_path.child('settings.py'))
from settings import *
from template import HTML_MAIL
now = datetime.datetime.utcnow()
gap = datetime.timedelta(days=INTERVAL_DAYS)
begins = now - gap
class ReleaseUnpacker(object):
def __init__(self, release_search_dir, tmp_dir, unpack_dir,
no_remove=False):
self.release_search_dir = Path(release_search_dir)
self.release_search_dir_abs = self.release_search_dir.absolute()
self.tmp_dir = Path(tmp_dir)
self.unpack_dir = Path(unpack_dir)
self.no_remove = no_remove
if not self.release_search_dir_abs.exists():
raise ReleaseUnpackerError(
'Release search dir {} doesn\'t exist'.format(
self.release_search_dir))
elif not self.release_search_dir_abs.isdir():
raise ReleaseUnpackerError(
'Release search dir {} is not a dir'.format(
self.release_search_dir))
elif not self.tmp_dir.exists():
raise ReleaseUnpackerError(
'Tmp dir {} doesn\'t exist'.format(self.tmp_dir))
elif not self.tmp_dir.isdir():
raise ReleaseUnpackerError(
'Tmp dir {} is not a dir'.format(
self.tmp_dir))
elif not self.unpack_dir.exists():
raise ReleaseUnpackerError(
'Unpack dir {} doesn\'t exist'.format(self.unpack_dir))
elif not self.unpack_dir.isdir():
raise ReleaseUnpackerError(
'Unpack dir {} is not a dir'.format(
self.unpack_dir))
def __repr__(self):
return '<ReleaseUnpacker: {} ({}) ({})>'.format(
self.release_search_dir_abs, self.tmp_dir, self.unpack_dir)
def file_exists_size_match(self, unpack_file_path, size_in_rar):
"""Returns True if unpack_file_path exists and size is a match"""
if (unpack_file_path.exists() and
unpack_file_path.size() == size_in_rar):
log.info('{} already exists and size match'.format(
unpack_file_path))
return True
else:
return False
def scan_rars(self):
"""Find all folders in release_search_dir and return the first RAR
file if one is found in a dir.
"""
rar_files = []
scan_dirs = [dir for dir in
self.release_search_dir_abs.walk(filter=DIRS)]
scan_dirs.append(self.release_search_dir_abs)
for dir in scan_dirs:
rar_files_found = dir.listdir(pattern='*.rar', filter=FILES)
if rar_files_found:
rar_files.append(rar_files_found[0])
return rar_files
def unpack_release_dir_rars(self):
"""Run the unpacker. Find the first RAR file in dirs found in
release_search_dir.
"""
# Scan for RAR files
self.rar_files = self.scan_rars()
if not self.rar_files:
log.debug('No RARs found in {}'.format(
self.release_search_dir_abs))
return False
# Process the RAR files in any were found
for rar_file_path in self.rar_files:
log.debug('Found RAR file {}'.format(rar_file_path))
release_unpacker_rar_file = ReleaseUnpackerRarFile(rar_file_path)
if release_unpacker_rar_file.subs_dir:
self.unpack_subs_rar(release_unpacker_rar_file)
else:
self.unpack_rar(release_unpacker_rar_file)
# Remove release dirs when unpack is done
self.remove_release_dirs()
return self
def remove_release_dirs(self):
"""Remove all release dirs from rar_files list"""
for rar_file_path in self.rar_files:
release_dir = rar_file_path.parent
if release_dir.exists():
if self.no_remove:
log.info('No remove active, not removing {}'.format(
release_dir))
else:
log.info(
'Unpack complete, removing {}'.format(release_dir))
release_dir.rmtree()
def unpack_subs_rar(self, release_unpacker_rar_file):
"""Unpack a RAR in a Subs folder"""
log.debug('Processing subs RAR file {}'.format(
release_unpacker_rar_file.rar_file_path_abs))
for rarfile_file in release_unpacker_rar_file.file_list:
# File in RAR is not a RAR file, extract
if rarfile_file['name'].ext != '.rar':
unpack_filename = '{}{}'.format(release_unpacker_rar_file.name,
rarfile_file['name'].ext)
unpack_file_path_abs = Path(self.unpack_dir, unpack_filename)
# File exists and size match
if self.file_exists_size_match(unpack_file_path_abs,
rarfile_file['size']):
continue
self.unpack_move_rar_file(release_unpacker_rar_file,
rarfile_file['name'],
unpack_file_path_abs)
# RAR file in RAR, extract to Subs folder and extract RAR
else:
log.debug('RAR file {} in {}'.format(
rarfile_file['name'],
release_unpacker_rar_file.rar_file_path_abs))
# Extract the RAR to Subs folder
subs_dir = release_unpacker_rar_file.rar_file_path_abs.parent
log.debug('Extracting {} to {}'.format(rarfile_file['name'],
subs_dir))
extracted_file_path = release_unpacker_rar_file.extract_file(
rarfile_file['name'], subs_dir)
# Extract the extracted Subs RAR file
self.unpack_subs_rar(ReleaseUnpackerRarFile(
extracted_file_path))
# Remove RAR file in Subs folder
log.debug('Removing extracted Subs RAR {}'.format(
extracted_file_path))
extracted_file_path.remove()
def unpack_rar(self, release_unpacker_rar_file):
"""List all files in a RAR and determine if it should be extracted
or not.
"""
for rarfile_file in release_unpacker_rar_file.file_list:
# Check file ext
if rarfile_file['name'].ext not in ('.avi', '.mkv',
'.img', '.iso',
'.mp4'):
log.info('Skipping {}, unwanted ext'.format(
rarfile_file['name']))
continue
unpack_filename = '{}{}'.format(release_unpacker_rar_file.name,
rarfile_file['name'].ext)
unpack_file_path_abs = Path(self.unpack_dir, unpack_filename)
# File exists and size match
if self.file_exists_size_match(unpack_file_path_abs,
rarfile_file['size']):
continue
# Unpack file in RAR
self.unpack_move_rar_file(release_unpacker_rar_file,
rarfile_file['name'],
unpack_file_path_abs)
return True
def unpack_move_rar_file(self, release_unpacker_rar_file,
rarfile_file_name, unpack_file_path):
"""Extract an individual file from release_unpacker_rar_file to
unpack_file_path
"""
# Extract file to tmp_dir
log.debug('Extracting {} to {}'.format(rarfile_file_name,
self.tmp_dir))
log.info('{} unpack started'.format(unpack_file_path.name))
unpack_start = datetime.now().replace(microsecond=0)
extracted_file_path = release_unpacker_rar_file.extract_file(
rarfile_file_name, self.tmp_dir)
unpack_end = datetime.now().replace(microsecond=0)
unpack_time = human(unpack_end - unpack_start, past_tense='{}')
if not unpack_time:
log.info('{} unpack done'.format(unpack_file_path.name))
else:
log.info('{} unpack done, {}'.format(unpack_file_path.name,
unpack_time))
# Move file and rename to unpack_dir
log.debug('Moving {} to {}'.format(extracted_file_path,
unpack_file_path))
extracted_file_path.move(unpack_file_path)