def __init__(self, conf, sub):
self.conf = conf
self.sub = sub
self.status = 0
self.sub_dir = os.path.join(self.conf.paths.base_dir,\
self.sub.title.text)
self.outcome = files.check_path(self.sub_dir)
if not self.outcome.success:
return
# merge sub settings and defaults
defaults = deepcopy(self.conf.xml.defaults)
rename = deepcopy(self.sub.rename) if hasattr(self.sub, 'rename') \
else None
errors = merge(self.sub, defaults, self.conf.xml.defaults, errors=[])
self.outcome = errors[0] if errors else Outcome(True, '')
defaults.tag = "subscription"
self.sub = defaults
if rename is not None:
self.sub.rename = rename
if not self.outcome.success:
return
# get jar and check for user deleted files
self.udeleted = []
self.jar, self.outcome = history.get_subjar(self.conf.paths,
self.sub)
if not self.outcome.success:
return
self.check_jar()
if not self.outcome.success:
return
# get feed, combine with jar and filter the lot
feed = Feed(self.sub, self.jar, self.udeleted)
self.status = feed.status
if self.status == 301:
self.outcome = Outcome(True, 'Feed has moved. Config updated.')
self.new_url = feed.href
elif self.status == 304:
self.outcome = Outcome(True, 'Not modified')
return
elif self.status >= 400:
self.outcome = Outcome(False, feed.bozo_exception)
return
else:
self.outcome = Outcome(True, 'Success')
combo = Combo(feed, self.jar, self.sub)
self.wanted = Wanted(self.sub, feed, combo, self.jar.del_lst,
self.sub_dir)
self.outcome = self.wanted.outcome
if not self.outcome.success:
return
from_the_top = self.sub.find('from_the_top') or 'no'
if from_the_top == 'no':
self.wanted.lst.reverse()
# subupgrade will delete unwanted and download lacking
self.unwanted = [x for x in self.jar.lst if x not in self.wanted.lst]
self.lacking = [x for x in self.wanted.lst if x not in self.jar.lst]
def delete_file(file_path):
'''Deletes a file'''
try:
os.remove(file_path)
return Outcome(True, file_path + ': File was successfully deleted')
except OSError as e:
return Outcome(False, 'Could not delete %s' % file_path)
def download_img_file(url, sub_dir, settings):
'''Download an image file'''
try:
r = requests.get(url, timeout=60)
except requests.exceptions.RequestException:
return Outcome(False, 'Download of %s failed' % url)
content_type = r.headers['content-type'].lower()
mime_dic = {'image/bmp': '.bmp',
'image/gif': '.gif',
'image/jpeg': '.jpg',
'image/jpg': '.jpg',
'image/png': '.png',
'image/webp': '.webp'}
extension = mime_dic.get(content_type, None)
if extension is None:
return Outcome(False, 'Download of image failed. Unknown MIME type.')
file_path = os.path.join(sub_dir, 'cover' + extension)
if os.path.isfile(file_path):
file_size = str(os.path.getsize(file_path))
remote_size = r.headers.get('content-length', str(len(r.content)))
if file_size == remote_size:
return Outcome(True, 'Same image file already downloaded')
with open(file_path, 'wb') as f:
f.write(r.content)
return Outcome(True, 'Image file downloaded')
def limit(self, sub):
'''Limit the number of episodes to that set in max_number'''
try:
self.lst = self.lst[:int(sub.max_number)]
self.outcome = Outcome(True, 'Number limited successfully')
except ValueError:
self.outcome = Outcome(False, 'Bad max_number setting')
def save(self):
'''Saves jar instance to file using pickle'''
try:
with open(self.db_filename, 'wb') as f:
pickle.dump(self, f)
outcome = Outcome(True, 'Pickle successful')
except (pickle.PickleError, PermissionError, FileNotFoundError,
IsADirectoryError):
outcome = Outcome(
False, 'Could not save history to %s' % self.db_filename)
return outcome
def check_file_write(check_file):
'''Check to see if file is writable/can be created'''
if os.path.isfile(check_file):
if os.access(check_file, os.W_OK):
return Outcome(True, '%s exists and is writable' % check_file)
else:
return Outcome(False, '%s exists but is not writable' % check_file)
if os.path.isdir(check_file):
return Outcome(False, '%s is a directory, not a file' % check_file)
outcome = check_path(os.path.dirname(check_file))
return outcome
def check_path(check_dir):
'''Create a directory'''
if os.path.isdir(check_dir):
if os.access(check_dir, os.W_OK):
return Outcome(True, '%s exists already' % check_dir)
else:
return Outcome(False, 'Could not save files to %s' % check_dir)
try:
os.makedirs(check_dir)
return Outcome(True, '%s was successfully created' % check_dir)
except OSError:
return Outcome(False, 'Could not create %s' % check_dir)
def validate_keys(audio_type, frames):
'''Returns a tuple with valid overrides and a list of invalid keys'''
valid_keys = type_dic.get(audio_type, None)
if valid_keys is None:
outcome = Outcome(False, 'Unsupported file type for tagging')
return (outcome, [], [])
overrides = [(override.tag, override.text) for override in frames
if override.tag in valid_keys(override.tag)]
invalid_keys = [
override.tag for override in frames
if override.tag not in valid_keys(override.tag)
]
outcome = Outcome(True, 'Supported file type for tagging')
return (outcome, overrides, invalid_keys)
def download_file(entry, settings):
'''Download function with block time outs'''
my_thread = current_thread()
headers = requests.utils.default_headers()
url = entry['poca_url']
if settings.useragent.text:
useragent = {'User-Agent': settings.useragent.text}
headers.update(useragent)
if getattr(my_thread, "kill", False):
return Outcome(None, 'Download cancelled by user')
try:
r = requests.get(url, stream=True, timeout=60, headers=headers)
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as e:
return Outcome(False, 'Download of %s failed' % url)
except requests.exceptions.Timeout:
return Outcome(False, 'Download of %s timed out' % url)
if r.status_code >= 400:
return Outcome(False, 'Download of %s failed' % url)
filename_keys = ['permissive', 'ntfs', 'restrictive', 'fallback']
start_at = settings.filenames.text or 'permissive'
if start_at in filename_keys:
filename_keys = filename_keys[filename_keys.index(start_at):]
if not entry['unique_filename']:
filename_keys = ['fallback']
for key in filename_keys:
filename = '.'.join((entry['names'][key], entry['extension']))
file_path = os.path.join(entry['directory'], filename)
try:
with open(file_path, 'wb') as f:
try:
for chunk in r.iter_content(chunk_size=1024):
if getattr(my_thread, "kill", False):
r.close()
_outcome = delete_file(f.name)
return Outcome(None, 'Download cancelled by user')
if chunk:
f.write(chunk)
r.close()
return Outcome(True, (filename, file_path))
except requests.exceptions.ConnectionError as e:
r.close()
_outcome = delete_file(f.name)
return Outcome(False, 'Download of %s broke off' % url)
except requests.exceptions.Timeout:
r.close()
_outcome = delete_file(f.name)
return Outcome(False, 'Download of %s timed out' % url)
except OSError:
#print('%s did not work, trying another...' % file_path)
pass
# testing
# this should really never happen
return Outcome(False, 'Somehow none of the filenames we tried worked')
def set_entries(self, doc, sub):
'''Extract entries from the feed xml'''
try:
self.lst = [entry.id for entry in doc.entries]
self.dic = {entry.id: entry for entry in doc.entries}
except (KeyError, AttributeError):
try:
self.lst = [
entry.enclosures[0]['href'] for entry in doc.entries
]
self.dic = {
entry.enclosures[0]['href']: entry
for entry in doc.entries
}
except (KeyError, AttributeError):
self.outcome = Outcome(False, 'Cant find entries in feed.')
# should we set an artificial status here? Or does feedparser?
# return
from_the_top = sub.find('from_the_top') or 'no'
if from_the_top == 'yes':
self.lst.reverse()
try:
self.image = doc.feed.image['href']
except (AttributeError, KeyError):
self.image = None
def check_path(check_dir):
'''Check directory exists and is writable; if not create directory'''
if os.path.isdir(check_dir):
if os.access(check_dir, os.W_OK):
return Outcome(True, '%s exists already' % check_dir)
else:
return Outcome(False, 'Could not save files to %s' % check_dir)
try:
os.makedirs(check_dir)
return Outcome(True, '%s was successfully created' % check_dir)
except FileExistsError:
return Outcome(False, 'Could not create %s. File already exists?' \
% check_dir)
except OSError:
return Outcome(False, 'Could not create %s. Illegal characters for ' \
'filesystem in directory name?' % check_dir)
def apply_filters(self, sub, combo):
'''Apply all filters set to be used on the subscription'''
func_dic = {'after_date': self.match_date,
'filename': self.match_filename,
'title': self.match_title,
'hour': self.match_hour,
'weekdays': self.match_weekdays}
filters = {node.tag for node in sub.filters.iterchildren()}
valid_filters = filters & set(func_dic.keys())
for key in valid_filters:
try:
func_dic[key](combo.dic, sub.filters[key].text)
self.outcome = Outcome(True, 'Filters applied successfully')
except KeyError as e:
self.outcome = Outcome(False, 'Entry is missing info: %s' % e)
except (ValueError, TypeError, SyntaxError) as e:
self.outcome = Outcome(False, 'Bad filter setting: %s' % e)
def write(conf):
'''Writes the resulting conf file back to poca.xml'''
root_str = pretty_print(conf.xml)
conf_file = conf.paths.config_file
test = os.access(conf_file, os.R_OK) and os.access(conf_file, os.W_OK)
if not test:
return Outcome(False, 'Lacking permissions to update config file')
with open(conf.paths.config_file, 'r+') as f:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
f.seek(0)
f.truncate()
f.write(root_str)
fcntl.flock(f, fcntl.LOCK_UN)
return Outcome(True, 'Config file updated')
except BlockingIOError:
return Outcome(False, 'Config file blocked')
def save(self):
'''Saves jar instance to file using pickle'''
outcome = files.check_path(os.path.dirname(self.db_filename))
if outcome.success:
with open(self.db_filename, 'wb') as f:
pickle.dump(self, f)
outcome = Outcome(True, 'Pickle successful')
return outcome
def open_jar(db_filename):
'''Tries opening existing jar'''
try:
with open(db_filename, mode='rb') as f:
jar = pickle.load(f)
outcome = Outcome(True, 'Jar loaded')
except (PermissionError, pickle.UnpicklingError, EOFError) as e:
outcome = Outcome(False,
'Could not read history from %s' % db_filename)
jar = None
except ImportError:
outcome = Outcome(
False, 'Issue with db encountered. If you have '
'upgraded from pre-1.0 release,\nplease delete '
'your old database (%s)' % db_filename)
jar = None
return jar, outcome
def __init__(self, email, paths):
handlers.BufferingHandler.__init__(self, int(email.threshold))
self.state_jar, outcome = history.get_statejar(paths)
self.buffer = self.state_jar.buffer
self.outcome = Outcome(None, '')
self.email = email
smtp_formatter = logging.Formatter("%(asctime)s %(message)s",
datefmt='%Y-%m-%d %H:%M')
self.setFormatter(smtp_formatter)
def get_subjar(paths, sub):
'''Returns existing jar if any, else creates a new one'''
db_filename = os.path.join(paths.db_dir, sub.title.text)
if os.path.isfile(db_filename):
jar, outcome = open_jar(db_filename)
else:
jar = Subjar(paths, sub)
outcome = Outcome(True, 'New jar created')
if outcome.success is True:
jar.db_filename = db_filename
return jar, outcome
def __init__(self, sub, feed, combo, del_lst, sub_dir):
self.outcome = Outcome(True, 'Wanted entries assembled')
self.lst = combo.lst
self.lst = list(filter(lambda x: x not in del_lst, self.lst))
self.lst = list(filter(lambda x: combo.dic[x]['valid'], self.lst))
if hasattr(sub, 'filters'):
self.apply_filters(sub, combo)
if hasattr(sub, 'max_number'):
self.limit(sub)
self.dic = {
uid: entryinfo.expand(combo.dic[uid], sub, sub_dir)
for uid in self.lst
}
filename_set = {self.dic[uid]['poca_filename'] for uid in self.lst}
if len(filename_set) < len(self.lst):
self.outcome = Outcome(
False, "Filename used more than once. "
"Use rename tag to fix.")
self.feed_etag = feed.etag
self.feed_modified = feed.modified
self.feed_image = feed.image
def update_url(args, subdata):
'''Used to implement 301 status code changes into conf'''
pseudo_args = Namespace(title=subdata.sub.title, url=None)
conf = config.Config(args, merge_default=False)
sub = search(conf.xml, pseudo_args)[0]
sub.url = subdata.new_url
_outcome = write(conf)
move = 'Feed moved to %s. ' % subdata.new_url
msg = move + 'Successfully update config file' if _outcome.success \
else move + 'Failed to update config file. ' + \
'Check permissions or update manually.'
return Outcome(_outcome.success, msg)
def download_img_file(url, sub_dir, settings):
'''Download an image file'''
try:
r = requests.get(url, timeout=60)
except requests.exceptions.RequestException:
return Outcome(False, 'Download of %s failed' % url)
content_type = r.headers['content-type'].lower()
mime_dic = {'image/bmp': '.bmp',
'image/gif': '.gif',
'image/jpeg': '.jpg',
'image/jpg': '.jpg',
'image/png': '.png',
'image/webp': '.webp'}
extension = mime_dic.get(content_type, None)
if extension is None:
return Outcome(False, 'Download of image %s failed. Unknown MIME type.'
% url)
else:
file_path = os.path.join(sub_dir, 'cover' + extension)
with open(file_path, 'wb') as f:
f.write(r.content)
return Outcome(True, '')
def flush(self):
'''Flush if we exceed threshold; otherwise save the buffer'''
if not self.buffer:
self.outcome = Outcome(None, 'Buffer was empty')
return
if len(self.buffer) < self.capacity:
self.outcome = Outcome(None, 'Buffer no sufficiently full')
self.save()
return
body = str()
for record in self.buffer:
body = body + self.format(record) + "\r\n"
msg = MIMEText(body.encode('utf-8'), _charset="utf-8")
msg['From'] = self.email.fromaddr.text
msg['To'] = self.email.toaddr.text
msg['Subject'] = Header("POCA log")
if self.email.starttls == 'yes':
try:
smtp = smtplib.SMTP(self.email.host.text, 587, timeout=10)
ehlo = smtp.ehlo()
except (ConnectionRefusedError, socket.gaierror, socket.timeout) \
as error:
self.outcome = Outcome(False, str(error))
self.save()
return
smtp.starttls()
try:
smtp.login(self.email.fromaddr.text, self.email.password.text)
except smtplib.SMTPAuthenticationError as error:
self.outcome = Outcome(False, str(error))
self.save()
return
else:
try:
smtp = smtplib.SMTP(self.email.host.text, 25, timeout=10)
ehlo = smtp.ehlo()
except (ConnectionRefusedError, socket.gaierror, socket.timeout) \
as error:
self.outcome = Outcome(False, str(error))
self.save()
return
try:
smtp.sendmail(self.email.fromaddr.text, [self.email.toaddr.text],
msg.as_string())
self.outcome = Outcome(True, "Succesfully sent email")
except (smtplib.SMTPException, socket.timeout) as error:
self.outcome = Outcome(False, str(error))
self.save()
return
smtp.quit()
self.buffer = []
self.state_jar.buffer = self.buffer
self.state_jar.save()
def write_config_file(config_file_path):
'''Writes default config xml to config file'''
print("No config file found. Making one at %s." % config_file_path)
default_base_dir = path.expanduser(path.join('~', 'poca'))
query = input("Please enter the full path for placing media files.\n"
"Press Enter to use default (%s): " % default_base_dir)
template_file = StringIO(TEMPLATE)
config_xml = objectify.parse(template_file)
config_root = config_xml.getroot()
config_root.settings.base_dir = query if query else default_base_dir
config_xml_str = pretty_print(config_xml)
try:
config_file = open(config_file_path, mode='wt', encoding='utf-8')
config_file.write(config_xml_str)
config_file.close()
msg = ("Default config succesfully written to %s.\n"
"Please edit or run 'poca-subscribe' to add subscriptions."
% config_file_path)
return Outcome(True, msg)
except IOError as e:
msg = "Failed writing config to %s.\nError: %s" % (config_file_path,
str(e))
return Outcome(False, msg)
def __init__(self, sub, feed, combo, del_lst, sub_dir):
self.outcome = Outcome(True, 'Default true')
self.lst = combo.lst
self.lst = list(filter(lambda x: x not in del_lst, self.lst))
self.lst = list(filter(lambda x: combo.dic[x]['valid'], self.lst))
if hasattr(sub, 'filters'):
self.apply_filters(sub, combo)
if hasattr(sub, 'max_number'):
self.limit(sub)
self.dic = {uid: entryinfo.expand(combo.dic[uid], sub, sub_dir)
for uid in self.lst}
filenames = [self.dic[uid]['poca_filename'] for uid in self.lst]
for uid in self.lst:
count = filenames.count(self.dic[uid]['poca_filename'])
if count > 1:
self.dic[uid]['unique_filename'] = False
else:
self.dic[uid]['unique_filename'] = True
self.feed_etag = feed.etag
self.feed_modified = feed.modified
self.feed_image = feed.image
def merge(user_el, new_el, default_el, errors=[]):
'''Updating one lxml objectify elements with another
(with primitive validation)'''
for child in user_el.iterchildren():
new_child = new_el.find(child.tag)
default_child = default_el.find(child.tag)
if default_child is None:
new_el.append(child)
continue
if isinstance(child, objectify.ObjectifiedDataElement):
right_type = type(child) == type(default_child)
valid = child.text in default_child.attrib.values() \
if default_child.attrib else True
if all((right_type, valid)):
new_el.replace(new_child, child)
else:
errors.append(Outcome(False, '%s: %s. Value not valid'
% (child.tag, child.text)))
elif isinstance(child, objectify.ObjectifiedElement):
merge(child, new_child, default_child, errors=errors)
return errors
def download_file(url, file_path, settings):
'''Download function with block time outs'''
my_thread = current_thread()
headers = requests.utils.default_headers()
if settings.useragent.text:
useragent = {'User-Agent': settings.useragent.text}
headers.update(useragent)
if getattr(my_thread, "kill", False):
return Outcome(None, 'Download cancelled by user')
try:
r = requests.get(url, stream=True, timeout=60, headers=headers)
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as e:
return Outcome(False, 'Download of %s failed' % url)
except requests.exceptions.Timeout:
return Outcome(False, 'Download of %s timed out' % url)
if r.status_code >= 400:
return Outcome(False, 'Download of %s failed' % url)
with open(file_path, 'wb') as f:
try:
for chunk in r.iter_content(chunk_size=1024):
if getattr(my_thread, "kill", False):
r.close()
_outcome = delete_file(f.name)
return Outcome(None, 'Download cancelled by user')
if chunk:
f.write(chunk)
except requests.exceptions.ConnectionError as e:
r.close()
_outcome = delete_file(f.name)
return Outcome(False, 'Download of %s broke off' % url)
except requests.exceptions.Timeout:
r.close()
_outcome = delete_file(f.name)
return Outcome(False, 'Download of %s timed out' % url)
r.close()
return Outcome(True, '')
def tag_audio_file(settings, sub, jar, entry):
'''Metdata tagging using mutagen'''
id3v1_dic = {'yes': 0, 'no': 2}
id3v1 = id3v1_dic[settings.id3removev1.text]
id3v2 = int(settings.id3v2version)
tracks = sub.find('./track_numbering')
tracks = tracks.text if tracks else 'no'
frames = sub.xpath('./metadata/*')
invalid_keys = []
if not frames and tracks == 'no':
return Outcome(True, 'Tagging skipped')
# get access to metadata
try:
audio = mutagen.File(entry['poca_abspath'])
except mutagen.MutagenError:
return Outcome(False, '%s not found or invalid file type for tagging'
% entry['poca_abspath'])
except mutagen.mp3.HeaderNotFoundError:
return Outcome(False, '%s bad mp3'
% entry['poca_abspath'])
if audio is None:
return Outcome(False, '%s is invalid file type for tagging' %
entry['poca_abspath'])
if audio.tags is None:
audio.add_tags()
# easify
if isinstance(audio, mutagen.mp3.MP3):
if id3v2 == 3:
audio.tags.update_to_v23()
elif id3v2 == 4:
audio.tags.update_to_v24()
audio.save(v1=id3v1, v2_version=id3v2)
audio = mutagen.File(entry['poca_abspath'], easy=True)
if isinstance(audio, mutagen.mp4.MP4):
audio = mutagen.File(entry['poca_abspath'], easy=True)
# validate the fields and file type
audio_type = type(audio)
outcome, overrides, invalid_keys = validate_keys(audio_type, frames)
if outcome.success is False:
return outcome
# run overrides
for override in overrides:
audio[override[0]] = override[1]
if tracks == 'yes' or (tracks == 'if missing' and 'tracknumber' not in
audio):
track_no = jar.track_no if hasattr(jar, 'track_no') else 0
track_no += 1
jar.track_no = track_no
jar.save()
if isinstance(audio, (mutagen.mp3.EasyMP3, mutagen.oggvorbis.OggVorbis,
mutagen.oggopus.OggOpus, mutagen.flac.FLAC)):
audio['tracknumber'] = str(track_no)
# else?
# save and finish
if isinstance(audio, mutagen.mp3.EasyMP3):
audio.save(v1=id3v1, v2_version=id3v2)
else:
audio.save()
if not invalid_keys:
return Outcome(True, 'Metadata successfully updated')
else:
return Outcome(False, '%s is set to add invalid tags: %s' %
(sub.title.text, ', '.join(invalid_keys)))
def tag_audio_file(settings, sub, jar, entry):
'''Metadata tagging using mutagen'''
# id3 settings
id3v1 = id3v1_dic[settings.id3removev1.text]
id3v2 = int(settings.id3v2version)
id3encoding = encodings[id3v2]
# overrides
frames = sub.xpath('./metadata/*')
overrides = [(override.tag, override.text) for override in frames]
key_errors = {}
# track numbering
tracks = sub.find('./track_numbering')
tracks = tracks.text if tracks else 'no'
if not overrides and tracks == 'no':
return Outcome(True, 'Tagging skipped')
# get 'easy' access to metadata
try:
audio = mutagen.File(entry['poca_abspath'], easy=True)
except mutagen.MutagenError:
return Outcome(
False, '%s not found or invalid file type for tagging' %
entry['poca_abspath'])
except mutagen.mp3.HeaderNotFoundError:
return Outcome(False, '%s is a bad mp3' % entry['poca_abspath'])
if audio is None:
return Outcome(
False,
'%s is invalid file type for tagging' % entry['poca_abspath'])
# add_tags is undocumented for easy but seems to work
if audio.tags is None:
audio.add_tags()
# tracks
if tracks == 'yes' or (tracks == 'if missing'
and 'tracknumber' not in audio):
track_no = jar.track_no if hasattr(jar, 'track_no') else 0
track_no += 1
overrides.append(('tracknumber', str(track_no)))
jar.track_no = track_no
jar.save()
# run overrides and save
while overrides:
tag, text = overrides.pop()
if not text and tag in audio:
_text = audio.pop(tag)
continue
try:
audio[tag] = text
except (easyid3.EasyID3KeyError, easymp4.EasyMP4KeyError, \
ValueError) as e:
key_errors[tag] = text
audio.save()
# ONLY for ID3
if isinstance(audio, mutagen.mp3.EasyMP3):
audio = mutagen.File(entry['poca_abspath'], easy=False)
if 'comment' in key_errors:
audio.tags.delall('COMM')
comm_txt = key_errors.pop('comment')
if comm_txt:
comm = id3.COMM(encoding=id3encoding, lang='eng', \
desc='desc', text=comm_txt)
audio.tags.add(comm)
if 'chapters' in key_errors:
_toc = key_errors.pop('chapters')
audio.tags.delall('CTOC')
audio.tags.delall('CHAP')
if id3v2 == 3:
audio.tags.update_to_v23()
elif id3v2 == 4:
audio.tags.update_to_v24()
audio.save(v1=id3v1, v2_version=id3v2)
# invalid keys
invalid_keys = list(key_errors.keys())
if not invalid_keys:
return Outcome(True, 'Metadata successfully updated')
else:
return Outcome(
False, '%s is set to add invalid tags: %s' %
(sub.title.text, ', '.join(invalid_keys)))
def verify_file(entry):
'''Check to see if recorded file exists or has been removed'''
isfile = os.path.isfile(entry['poca_abspath'])
return Outcome(isfile, entry['poca_abspath'] + ' exists: ' + str(isfile))
