def rename_file(self, path: str, template: str, sanitize: bool = True):
"""
Renames a given file according to a given template.
Args:
path (str): Path to the file to rename.
template (str): Output template to rename the file by.
sanitize (bool, optional): Defaults to True. True to
replace path delimiters within the substituted
template to avoid unexpected directories.
Returns:
str: The renamed path to the file.
"""
if not path.exists():
return
info = FileSystem(path = path)
replaced_template = self.replace_values(info, template)
if sanitize:
replaced_template = replaced_template.replace('/', '_').replace('\\', '_')
new_filename = path.with_name(replaced_template)
path.rename(new_filename)
Logger.log(r'PostProcessor', r'{} => {}'.format(path.name, replaced_template))
return new_filename
def construct_command(self, subscription: Subscription) -> str:
"""
Builds the youtube-dl command for the given subscription.
Args:
subscription (Subscription): The subscription to process.
Returns:
str: The youtube-dl command with all desired arguments.
"""
command = r'youtube-dl'
# Add the youtube-dl config path.
if subscription.youtubedl_config.config:
config_path = os.path.join(os.getenv('youtubedl_config_directory'), subscription.youtubedl_config.config)
command += r' --config-location "{}"'.format(config_path)
# Add the metadata-from-title pattern.
if subscription.youtubedl_config.metadata_format:
command += r' --metadata-from-title "{}"'.format(subscription.youtubedl_config.metadata_format)
# Add the output pattern.
if subscription.youtubedl_config.output_format:
output_format = subscription.staging_directory + '/staging_area/' + subscription.youtubedl_config.output_format
command += r' -o "{}"'.format(output_format)
# Add the path to the video ID archive.
if subscription.youtubedl_config.archive:
archive_path = os.path.join(subscription.staging_directory, subscription.youtubedl_config.archive)
command += r' --download-archive "{}"'.format(archive_path)
# Add any extra arguments this sub has.
if subscription.youtubedl_config.extra_commands:
command += " " + subscription.youtubedl_config.extra_commands
# Add the subscription URL.
command += r' "{}"'.format(subscription.url)
# Construct the post-processing call back into
# Chrysalis to be run after each successful download.
if subscription.post_processing:
command += ' --exec \'"{}" "{}" --postprocess {{}} --subscription "{}"\''.format(
sys.executable,
__file__,
subscription.name
)
# Construct the stdout redirect to the log file.
if subscription.logging.path:
command += r' {} "{}"'.format(
'>>' if subscription.logging.append == True else '>',
subscription.logging.path
)
Logger.log(r'Chrysalis', r'Command to be run: [{}]'.format(command))
return command
def run(self, postprocessor):
Logger.log(r'Plex',
r'Processing {}...'.format(postprocessor.path_info.parent),
1)
self.postprocessor = postprocessor
self.login()
if "metadata" in self.postprocessor.settings.post_processing.destination:
self.set_metadata()
def run(self):
"""
Runs all post-processing on the file.
Returns:
str: Path to the post-processed output directory.
"""
self.path_info = pathlib.Path(self.file)
Logger.log(r'PostProcessor', r'Processing {}...'.format(self.path_info.parent), 1)
if self.settings.post_processing.repositories is not None and 'tvdb' in self.settings.post_processing.repositories:
self.get_api_info()
if self.settings.post_processing.pattern is not None:
self.special_values = re.match(self.settings.post_processing.pattern, str(self.path_info))
if self.settings.post_processing.subtitle is not None:
self.rename_subtitles()
if self.settings.post_processing.thumbnail is not None:
self.rename_thumbnails()
if self.settings.post_processing.description is not None:
self.rename_description()
if self.settings.post_processing.video is not None:
self.path_info = self.rename_video()
if self.settings.post_processing.episode_folder is not None:
new_folder = self.rename_folder()
else:
new_folder = self.path_info.parent
self.path_info = new_folder / self.path_info.name
if self.settings.post_processing.output_directory:
self.move_from_staging_area(new_folder)
if self.settings.post_processing.real_destinations:
destination = self.settings.post_processing.real_destinations[0]
if destination:
destination.run(self)
Logger.tabs -= 1
return new_folder
def get_series_episodes(self, series_id: int):
"""
Query episode information for series.
Automatically retrieves all pages.
Args:
series_id (int): Unique ID of the series.
Returns:
list: List of dicts for episodes.
"""
Logger.log(r'API', r'Querying episodes...')
page = 1
episodes = []
while True:
response = self.http.request(
'GET',
self.base_url + 'series/' + str(series_id) +
'/episodes?page=' + str(page),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + self.jwt
})
response_data = json.loads(response.data.decode('utf-8'))
episodes.extend(response_data['data'])
if response_data['links']['next'] is None or page == response_data[
'links']['last']:
break
page += 1
episodes = list({
v['episodeName']: v
for v in episodes
if v['episodeName'] is not None and v['episodeName'] != ''
}.values())
return episodes
def try_description_file(self):
description_file_path = self.postprocessor.path_info.with_suffix(
'.description')
try:
with open(description_file_path, 'r') as myfile:
self.metadata['summary']['found value'] = myfile.read()
except:
Logger.log(r'Plex', r'Couldn\'t read from *.description file.')
description_file_path = self.postprocessor.path_info.parent / 'description.txt'
try:
with open(description_file_path, 'r') as myfile:
self.metadata['summary']['found value'] = myfile.read()
except:
Logger.log(r'Plex', r'Couldn\'t read from description.txt file.')
def move_from_staging_area(self, current_path):
"""
Moves the post-processed folder from the staging area
to the final path specified by the subscription.
"""
out_dir = self.settings.post_processing.output_directory
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
final_dir = out_dir + '/' + current_path.name
current_path.rename(final_dir)
self.path_info = pathlib.Path(final_dir) / self.path_info.name
Logger.log(r'PostProcessor', r'Refoldered to {}'.format(final_dir))
def __call__(self, **kwargs):
"""Start the clients in parallel"""
[
setattr(self, k, kwargs.pop(k)) for k in list(kwargs)
if k in self.__dict__
]
if len(kwargs):
raise TypeError(
"{} got one or more unexpected arguments {}".format(
"Clients.__call__", list(kwargs.keys())))
if self.nclients == 0:
return
if not self.nclients:
self.nclients = cpu_count()
if not self.logger:
self.logger = Logger(fmt=FMT_LONG,
name='Clients',
level=self.loglevel)
self.logger.info("Starting {} Clients".format(self.nclients))
opts = {
k: self.__dict__[k]
for k in ('server', 'password', 'port', 'timeout', 'loglevel')
}
self.clients = []
try:
for i in range(self.nclients):
client = Client(**opts)
client.start()
self.clients.append(client)
except:
exc_info = sys.exc_info()
self.logger.exception("{}".format(exc_info[1]))
return 1
return 0
def process_subscription(self, subscription: Subscription):
"""
Runs youtube-dl and the post-processing for the given subscription.
Parameters:
subscription (Subscription): The subscription to process.
"""
if not subscription.enabled:
return
Logger.log(r'Chrysalis', r'Processing "{}"...'.format(subscription.name))
self.setup_staging_directory(subscription)
if subscription.logging and subscription.logging.path:
pathlib.Path(subscription.logging.path).parent.mkdir(parents=True, exist_ok=True)
command = self.construct_command(subscription)
subprocess.run(command, shell=True)
def try_file_metadata(self):
try:
file_metadata = ffmpeg.probe(str(self.postprocessor.path_info))
except:
None
if file_metadata:
if self.metadata['summary']['wanted']:
try:
self.metadata['summary']['found value'] = file_metadata[
'format']['tags']['DESCRIPTION']
except:
Logger.log(r'Plex',
r'Couldn\'t read description from tags.')
if self.metadata['originallyAvailableAt']['wanted']:
try:
temp_date = file_metadata['format']['tags']['DATE']
self.metadata['originallyAvailableAt']['found value'] = temp_date[0:4] \
+ '-' + temp_date[4:6] + '-' + temp_date[6:8]
except:
Logger.log(r'Plex', r'Couldn\'t read date from tags.')
def get_series(self, series_id: int):
"""
Query series information.
Args:
series_id (int): Unique ID of the series.
Returns:
dict: Series infomation.
"""
Logger.log(r'API', r'Querying series...')
response = self.http.request('GET',
self.base_url + 'series/' +
str(series_id),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + self.jwt
})
response_data = json.loads(response.data.decode('utf-8'))
return response_data['data']
def postprocess(self, file: str, subscription: Subscription) -> str:
"""
Runs the post-processing for the given youtube-dl output file.
Args:
file (str): Absolute path to the youtube-dl output file.
subscription (Subscription): The settings to process the file under.
Returns:
str: The absolute path to the folder where all the files were moved.
"""
from PostProcessor import PostProcessor
Logger.log(r'Crysalis', r'Starting PostProcessor for {}'.format(file), 1)
postprocessor = PostProcessor(
file = file,
settings = subscription
)
postprocessor.run()
Logger.tabs -= 1
def login(self):
"""
Pass the user credentials to receive a JWT.
"""
Logger.log(r'API', r'Logging in...')
request_data = {
'username': self.username,
'userkey': self.user_key,
'apikey': self.api_key,
}
encoded_data = json.dumps(request_data).encode('utf-8')
response = self.http.request(
'POST',
self.base_url + 'login',
body=encoded_data,
headers={'Content-Type': 'application/json'})
response_data = json.loads(response.data.decode('utf-8'))
self.jwt = response_data['token']
def get_api_info(self):
"""
Retrieves the API info for the file.
"""
from Repositories.TVDB import TVDB
Logger.log(r'API', r'Querying...', 1)
api = TVDB()
api.login()
self.series = api.get_series(self.settings.post_processing.series_id)
episodes = api.get_series_episodes(self.settings.post_processing.series_id)
if episodes is None or len(episodes) == 0:
Logger.log(r'API', r'No episodes returned!', -1)
return -1
file_matches = re.match(self.settings.post_processing.pattern, str(self.path_info))
self.episode = api.match_episode(episodes, file_matches.group('episodeName'))
Logger.tabs -= 1
def set_metadata(self):
Logger.log(r'Plex', r'Setting metadata...', 1)
self.parse_metadata_settings()
episode = self.get_episode()
if not episode:
return -1
# Try to pull the metadata out of the file itself, first.
self.try_file_metadata()
# If we couldn't get the description from the file, try a .description file.
if not self.metadata['summary']['found value'] and self.metadata[
'summary']['wanted']:
self.try_description_file()
values = {}
for item in self.metadata:
if self.metadata[item]['found value']:
Logger.log(
r'Plex', r'Setting metadata: {} = {}...'.format(
item, self.metadata[item]['found value'][:20]))
values[item + '.value'] = self.metadata[item]['found value']
values[item + '.locked'] = 1
# Plex seems to have a race condition if you set values before it's
# finished the library scan. If the scan finishes afterwards, it seems
# like the values get reset.
episode.section().cancelUpdate()
time.sleep(10)
episode.edit(**values)
Logger.log(r'Plex', r'Metadata set.')
Logger.tabs -= 1
def __call__(self, **kwargs):
"""Run the server
While parameters may be passed to this method, the intention is that
the constructor will be used to create a closure setting the arguments.
This provides a means for the user to adjust the parameters prior to
calling the created object to run the server.
"""
# Process arguments
[
setattr(self, k, kwargs.pop(k)) for k in list(kwargs)
if k in self.__dict__
]
if len(kwargs):
raise TypeError(
"{} got one or more unexpected arguments {}".format(
"Clients.__call__", list(kwargs.keys())))
if type(self.password) == str:
self.password = self.password.encode("utf-8")
# Setup logging
if not self.logger:
self.logger = Logger(fmt=FMT_LONG,
name='Server',
level=self.loglevel)
# Save server pid
if 'MAPREDUCE_PID_FILE' in os.environ:
with open(os.environ['MAPREDUCE_PID_FILE'], 'w') as pf:
print(os.getpid(), file=pf)
# Setup signal handling
def SIGHANDLER(sig, frame):
"Handler for signal, just sets flag to close processing"
self.logger.warning(
"Signal {} detected, waiting for clients to finish".format(
sig))
self.fail = True
signal.signal(STOP_SIGNAL, SIGHANDLER)
# Start Clients
if self.nclients != 0:
c = Clients(nclients=self.nclients,
password=self.password,
port=self.port,
logger=self.logger,
loglevel=self.loglevel)
if c():
return 1
# Start Server
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(("", self.port))
self.listen(5)
try:
asyncore.loop(map=self.socket_map)
except:
self.close()
raise
# Close any remaining clients
if self.nclients != 0:
c.terminate()
if self.fail:
self.logger.warning("Server ending abnormally")
raise MapReduceError("Server ending abnormally")
return self.taskmanager.results
help='Number of clients to run',
type=int,
default=None)
loggerValues = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
p.add_argument('-l',
'--loglevel',
help='Logger level (debug, info, warning, error, critical)',
type=lambda x: x.upper(),
choices=loggerValues,
default='info')
opt = p.parse_args()
log = Logger(level=opt.loglevel, fmt=FMT_LONG, name='Clients')
log.info("Starting {}".format(sys.argv[0]))
# Show options
log.info(__doc__.splitlines()[1])
log.info("Options specified:")
for o in sorted(opt.__dict__.keys()):
log.info("{:12s} = {}".format(o, opt.__dict__[o]))
# Run the clients
clients = Clients(logger=log, **vars(opt))
rc = clients()
if not rc:
class Clients(object):
"""Run a set of clients as separate processes:
nclients is specified:
None : Run the number of clients sized to the number of cores (default)
0 : Run none
>0 : Run the pool of processes sized as specified
"""
def __init__(self,
nclients=None,
server=None,
password=None,
port=None,
timeout=None,
logger=None,
loglevel=None):
self.server = server if server else DEFAULT_HOST
self.port = port if port else DEFAULT_PORT
self.password = password if password else DEFAULT_PASSWORD
self.timeout = timeout if timeout else DEFAULT_TIMEOUT
self.nclients = nclients
self.logger = logger if logger else logging.getLogger()
self.loglevel = loglevel if loglevel else DEFAULT_LOG_LEVEL
def __call__(self, **kwargs):
"""Start the clients in parallel"""
[
setattr(self, k, kwargs.pop(k)) for k in list(kwargs)
if k in self.__dict__
]
if len(kwargs):
raise TypeError(
"{} got one or more unexpected arguments {}".format(
"Clients.__call__", list(kwargs.keys())))
if self.nclients == 0:
return
if not self.nclients:
self.nclients = cpu_count()
if not self.logger:
self.logger = Logger(fmt=FMT_LONG,
name='Clients',
level=self.loglevel)
self.logger.info("Starting {} Clients".format(self.nclients))
opts = {
k: self.__dict__[k]
for k in ('server', 'password', 'port', 'timeout', 'loglevel')
}
self.clients = []
try:
for i in range(self.nclients):
client = Client(**opts)
client.start()
self.clients.append(client)
except:
exc_info = sys.exc_info()
self.logger.exception("{}".format(exc_info[1]))
return 1
return 0
def terminate(self):
"""Forceably terminate the clients"""
[c.terminate for c in self.clients if c.is_alive()]
def join(self):
"""Wait for client completion"""
self.logger.debug("Waiting for clients to complete")
try:
for c in self.clients:
c.join()
self.logger.debug("Client ended with rc {}", c.exitcode)
except:
exc_info = sys.exc_info()
self.logger.exception("{}".format(exc_info[1]))
self.logger.info("All Clients Completed")
def get_episode(self):
destination_settings = self.postprocessor.settings.post_processing.destination
section_name = destination_settings["section"]
series_name = destination_settings[
"series"] or self.postprocessor.settings.name
if not section_name:
Logger.log(r'Plex', r'No "section" set!')
return None
if not series_name:
Logger.log(r'Plex', r'No "series" set!')
return None
section = self.session.library.section(section_name)
if not section:
Logger.log(r'Plex',
r'"{}" section not found!'.format(section_name))
return None
# Make sure Plex knows about our new video.
section.update()
time.sleep(3)
series = None
tries = 0
# Grab the show.
while not series and tries < 10:
try:
series = section.get(series_name)
except exceptions.NotFound:
# It can take plex a few seconds to
# find the show if it's new.
tries += 1
section.update()
time.sleep(10 * tries)
if not series:
Logger.log(r'Plex', r'"{}" series not found!'.format(series_name))
return None
episode = None
tries = 0
while not episode and tries < 10:
try:
episodes = series.episodes()
episodes = [episode for episode in episodes \
if pathlib.Path(episode.locations[0]).name == self.postprocessor.path_info.name]
episode = episodes[0]
except:
# It can take plex a few seconds to
# find the episode.
tries += 1
section.update()
time.sleep(10 * tries)
if not episode:
Logger.log('Plex', "Couldn't find episode!")
raise Exception("Couldn't find episode!")
return None
return episode
def login(self):
Logger.log(r'Plex', r'Logging in...', 1)
self.session = PlexServer(self.url, self.token)
Logger.tabs -= 1
class Server(asyncore.dispatcher, object):
"""Run the map/reduce server
It is recommended that all the appropriate parameters be passed to
the constructor.
If nclients is not specified a set of client prcesses will be started
equal to the number of CPU's available on the local machine.
"""
def __init__(self,
datasource,
mapfn,
reducefn,
collectfn=None,
port=None,
password=None,
logger=None,
loglevel=None,
nclients=None):
self.port = port if port else DEFAULT_PORT
self.password = password if password else DEFAULT_PASSWORD
self.logger = logger
self.nclients = nclients
self.loglevel = loglevel if loglevel else DEFAULT_LOG_LEVEL
self.socket_map = {}
asyncore.dispatcher.__init__(self, map=self.socket_map)
self.mapfn = mapfn
self.reducefn = reducefn
self.collectfn = collectfn
self.datasource = datasource
self._fail = False
def __call__(self, **kwargs):
"""Run the server
While parameters may be passed to this method, the intention is that
the constructor will be used to create a closure setting the arguments.
This provides a means for the user to adjust the parameters prior to
calling the created object to run the server.
"""
# Process arguments
[
setattr(self, k, kwargs.pop(k)) for k in list(kwargs)
if k in self.__dict__
]
if len(kwargs):
raise TypeError(
"{} got one or more unexpected arguments {}".format(
"Clients.__call__", list(kwargs.keys())))
if type(self.password) == str:
self.password = self.password.encode("utf-8")
# Setup logging
if not self.logger:
self.logger = Logger(fmt=FMT_LONG,
name='Server',
level=self.loglevel)
# Save server pid
if 'MAPREDUCE_PID_FILE' in os.environ:
with open(os.environ['MAPREDUCE_PID_FILE'], 'w') as pf:
print(os.getpid(), file=pf)
# Setup signal handling
def SIGHANDLER(sig, frame):
"Handler for signal, just sets flag to close processing"
self.logger.warning(
"Signal {} detected, waiting for clients to finish".format(
sig))
self.fail = True
signal.signal(STOP_SIGNAL, SIGHANDLER)
# Start Clients
if self.nclients != 0:
c = Clients(nclients=self.nclients,
password=self.password,
port=self.port,
logger=self.logger,
loglevel=self.loglevel)
if c():
return 1
# Start Server
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(("", self.port))
self.listen(5)
try:
asyncore.loop(map=self.socket_map)
except:
self.close()
raise
# Close any remaining clients
if self.nclients != 0:
c.terminate()
if self.fail:
self.logger.warning("Server ending abnormally")
raise MapReduceError("Server ending abnormally")
return self.taskmanager.results
_clientId = 0
def handle_accepted(self, conn, addr):
sc = ServerChannel(conn, addr, self.socket_map, Server._clientId, self)
Server._clientId += 1
def handle_close(self):
self.close()
def set_datasource(self, ds):
self._datasource = ds
self.taskmanager = TaskManager(self._datasource, self)
def get_datasource(self):
return self._datasource
datasource = property(get_datasource, set_datasource)
def get_fail(self):
return self._fail
def set_fail(self, v):
self._fail = v
if v:
self.taskmanager.state = TaskManager.FINISHED
fail = property(get_fail, set_fail)