def __init__(self, conf, sub): self.conf = conf self.sub = sub self.status = 0 self.sub_dir = os.path.join(self.conf.paths.base_dir,\ self.sub.title.text) self.outcome = files.check_path(self.sub_dir) if not self.outcome.success: return # merge sub settings and defaults defaults = deepcopy(self.conf.xml.defaults) rename = deepcopy(self.sub.rename) if hasattr(self.sub, 'rename') \ else None errors = merge(self.sub, defaults, self.conf.xml.defaults, errors=[]) self.outcome = errors[0] if errors else Outcome(True, '') defaults.tag = "subscription" self.sub = defaults if rename is not None: self.sub.rename = rename if not self.outcome.success: return # get jar and check for user deleted files self.udeleted = [] self.jar, self.outcome = history.get_subjar(self.conf.paths, self.sub) if not self.outcome.success: return self.check_jar() if not self.outcome.success: return # get feed, combine with jar and filter the lot feed = Feed(self.sub, self.jar, self.udeleted) self.status = feed.status if self.status == 301: self.outcome = Outcome(True, 'Feed has moved. Config updated.') self.new_url = feed.href elif self.status == 304: self.outcome = Outcome(True, 'Not modified') return elif self.status >= 400: self.outcome = Outcome(False, feed.bozo_exception) return else: self.outcome = Outcome(True, 'Success') combo = Combo(feed, self.jar, self.sub) self.wanted = Wanted(self.sub, feed, combo, self.jar.del_lst, self.sub_dir) self.outcome = self.wanted.outcome if not self.outcome.success: return from_the_top = self.sub.find('from_the_top') or 'no' if from_the_top == 'no': self.wanted.lst.reverse() # subupgrade will delete unwanted and download lacking self.unwanted = [x for x in self.jar.lst if x not in self.wanted.lst] self.lacking = [x for x in self.wanted.lst if x not in self.jar.lst]
def delete_file(file_path): '''Deletes a file''' try: os.remove(file_path) return Outcome(True, file_path + ': File was successfully deleted') except OSError as e: return Outcome(False, 'Could not delete %s' % file_path)
def download_img_file(url, sub_dir, settings): '''Download an image file''' try: r = requests.get(url, timeout=60) except requests.exceptions.RequestException: return Outcome(False, 'Download of %s failed' % url) content_type = r.headers['content-type'].lower() mime_dic = {'image/bmp': '.bmp', 'image/gif': '.gif', 'image/jpeg': '.jpg', 'image/jpg': '.jpg', 'image/png': '.png', 'image/webp': '.webp'} extension = mime_dic.get(content_type, None) if extension is None: return Outcome(False, 'Download of image failed. Unknown MIME type.') file_path = os.path.join(sub_dir, 'cover' + extension) if os.path.isfile(file_path): file_size = str(os.path.getsize(file_path)) remote_size = r.headers.get('content-length', str(len(r.content))) if file_size == remote_size: return Outcome(True, 'Same image file already downloaded') with open(file_path, 'wb') as f: f.write(r.content) return Outcome(True, 'Image file downloaded')
def limit(self, sub): '''Limit the number of episodes to that set in max_number''' try: self.lst = self.lst[:int(sub.max_number)] self.outcome = Outcome(True, 'Number limited successfully') except ValueError: self.outcome = Outcome(False, 'Bad max_number setting')
def save(self): '''Saves jar instance to file using pickle''' try: with open(self.db_filename, 'wb') as f: pickle.dump(self, f) outcome = Outcome(True, 'Pickle successful') except (pickle.PickleError, PermissionError, FileNotFoundError, IsADirectoryError): outcome = Outcome( False, 'Could not save history to %s' % self.db_filename) return outcome
def check_file_write(check_file): '''Check to see if file is writable/can be created''' if os.path.isfile(check_file): if os.access(check_file, os.W_OK): return Outcome(True, '%s exists and is writable' % check_file) else: return Outcome(False, '%s exists but is not writable' % check_file) if os.path.isdir(check_file): return Outcome(False, '%s is a directory, not a file' % check_file) outcome = check_path(os.path.dirname(check_file)) return outcome
def check_path(check_dir): '''Create a directory''' if os.path.isdir(check_dir): if os.access(check_dir, os.W_OK): return Outcome(True, '%s exists already' % check_dir) else: return Outcome(False, 'Could not save files to %s' % check_dir) try: os.makedirs(check_dir) return Outcome(True, '%s was successfully created' % check_dir) except OSError: return Outcome(False, 'Could not create %s' % check_dir)
def validate_keys(audio_type, frames): '''Returns a tuple with valid overrides and a list of invalid keys''' valid_keys = type_dic.get(audio_type, None) if valid_keys is None: outcome = Outcome(False, 'Unsupported file type for tagging') return (outcome, [], []) overrides = [(override.tag, override.text) for override in frames if override.tag in valid_keys(override.tag)] invalid_keys = [ override.tag for override in frames if override.tag not in valid_keys(override.tag) ] outcome = Outcome(True, 'Supported file type for tagging') return (outcome, overrides, invalid_keys)
def download_file(entry, settings): '''Download function with block time outs''' my_thread = current_thread() headers = requests.utils.default_headers() url = entry['poca_url'] if settings.useragent.text: useragent = {'User-Agent': settings.useragent.text} headers.update(useragent) if getattr(my_thread, "kill", False): return Outcome(None, 'Download cancelled by user') try: r = requests.get(url, stream=True, timeout=60, headers=headers) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: return Outcome(False, 'Download of %s failed' % url) except requests.exceptions.Timeout: return Outcome(False, 'Download of %s timed out' % url) if r.status_code >= 400: return Outcome(False, 'Download of %s failed' % url) filename_keys = ['permissive', 'ntfs', 'restrictive', 'fallback'] start_at = settings.filenames.text or 'permissive' if start_at in filename_keys: filename_keys = filename_keys[filename_keys.index(start_at):] if not entry['unique_filename']: filename_keys = ['fallback'] for key in filename_keys: filename = '.'.join((entry['names'][key], entry['extension'])) file_path = os.path.join(entry['directory'], filename) try: with open(file_path, 'wb') as f: try: for chunk in r.iter_content(chunk_size=1024): if getattr(my_thread, "kill", False): r.close() _outcome = delete_file(f.name) return Outcome(None, 'Download cancelled by user') if chunk: f.write(chunk) r.close() return Outcome(True, (filename, file_path)) except requests.exceptions.ConnectionError as e: r.close() _outcome = delete_file(f.name) return Outcome(False, 'Download of %s broke off' % url) except requests.exceptions.Timeout: r.close() _outcome = delete_file(f.name) return Outcome(False, 'Download of %s timed out' % url) except OSError: #print('%s did not work, trying another...' % file_path) pass # testing # this should really never happen return Outcome(False, 'Somehow none of the filenames we tried worked')
def set_entries(self, doc, sub): '''Extract entries from the feed xml''' try: self.lst = [entry.id for entry in doc.entries] self.dic = {entry.id: entry for entry in doc.entries} except (KeyError, AttributeError): try: self.lst = [ entry.enclosures[0]['href'] for entry in doc.entries ] self.dic = { entry.enclosures[0]['href']: entry for entry in doc.entries } except (KeyError, AttributeError): self.outcome = Outcome(False, 'Cant find entries in feed.') # should we set an artificial status here? Or does feedparser? # return from_the_top = sub.find('from_the_top') or 'no' if from_the_top == 'yes': self.lst.reverse() try: self.image = doc.feed.image['href'] except (AttributeError, KeyError): self.image = None
def check_path(check_dir): '''Check directory exists and is writable; if not create directory''' if os.path.isdir(check_dir): if os.access(check_dir, os.W_OK): return Outcome(True, '%s exists already' % check_dir) else: return Outcome(False, 'Could not save files to %s' % check_dir) try: os.makedirs(check_dir) return Outcome(True, '%s was successfully created' % check_dir) except FileExistsError: return Outcome(False, 'Could not create %s. File already exists?' \ % check_dir) except OSError: return Outcome(False, 'Could not create %s. Illegal characters for ' \ 'filesystem in directory name?' % check_dir)
def apply_filters(self, sub, combo): '''Apply all filters set to be used on the subscription''' func_dic = {'after_date': self.match_date, 'filename': self.match_filename, 'title': self.match_title, 'hour': self.match_hour, 'weekdays': self.match_weekdays} filters = {node.tag for node in sub.filters.iterchildren()} valid_filters = filters & set(func_dic.keys()) for key in valid_filters: try: func_dic[key](combo.dic, sub.filters[key].text) self.outcome = Outcome(True, 'Filters applied successfully') except KeyError as e: self.outcome = Outcome(False, 'Entry is missing info: %s' % e) except (ValueError, TypeError, SyntaxError) as e: self.outcome = Outcome(False, 'Bad filter setting: %s' % e)
def write(conf): '''Writes the resulting conf file back to poca.xml''' root_str = pretty_print(conf.xml) conf_file = conf.paths.config_file test = os.access(conf_file, os.R_OK) and os.access(conf_file, os.W_OK) if not test: return Outcome(False, 'Lacking permissions to update config file') with open(conf.paths.config_file, 'r+') as f: try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) f.seek(0) f.truncate() f.write(root_str) fcntl.flock(f, fcntl.LOCK_UN) return Outcome(True, 'Config file updated') except BlockingIOError: return Outcome(False, 'Config file blocked')
def save(self): '''Saves jar instance to file using pickle''' outcome = files.check_path(os.path.dirname(self.db_filename)) if outcome.success: with open(self.db_filename, 'wb') as f: pickle.dump(self, f) outcome = Outcome(True, 'Pickle successful') return outcome
def open_jar(db_filename): '''Tries opening existing jar''' try: with open(db_filename, mode='rb') as f: jar = pickle.load(f) outcome = Outcome(True, 'Jar loaded') except (PermissionError, pickle.UnpicklingError, EOFError) as e: outcome = Outcome(False, 'Could not read history from %s' % db_filename) jar = None except ImportError: outcome = Outcome( False, 'Issue with db encountered. If you have ' 'upgraded from pre-1.0 release,\nplease delete ' 'your old database (%s)' % db_filename) jar = None return jar, outcome
def __init__(self, email, paths): handlers.BufferingHandler.__init__(self, int(email.threshold)) self.state_jar, outcome = history.get_statejar(paths) self.buffer = self.state_jar.buffer self.outcome = Outcome(None, '') self.email = email smtp_formatter = logging.Formatter("%(asctime)s %(message)s", datefmt='%Y-%m-%d %H:%M') self.setFormatter(smtp_formatter)
def get_subjar(paths, sub): '''Returns existing jar if any, else creates a new one''' db_filename = os.path.join(paths.db_dir, sub.title.text) if os.path.isfile(db_filename): jar, outcome = open_jar(db_filename) else: jar = Subjar(paths, sub) outcome = Outcome(True, 'New jar created') if outcome.success is True: jar.db_filename = db_filename return jar, outcome
def __init__(self, sub, feed, combo, del_lst, sub_dir): self.outcome = Outcome(True, 'Wanted entries assembled') self.lst = combo.lst self.lst = list(filter(lambda x: x not in del_lst, self.lst)) self.lst = list(filter(lambda x: combo.dic[x]['valid'], self.lst)) if hasattr(sub, 'filters'): self.apply_filters(sub, combo) if hasattr(sub, 'max_number'): self.limit(sub) self.dic = { uid: entryinfo.expand(combo.dic[uid], sub, sub_dir) for uid in self.lst } filename_set = {self.dic[uid]['poca_filename'] for uid in self.lst} if len(filename_set) < len(self.lst): self.outcome = Outcome( False, "Filename used more than once. " "Use rename tag to fix.") self.feed_etag = feed.etag self.feed_modified = feed.modified self.feed_image = feed.image
def update_url(args, subdata): '''Used to implement 301 status code changes into conf''' pseudo_args = Namespace(title=subdata.sub.title, url=None) conf = config.Config(args, merge_default=False) sub = search(conf.xml, pseudo_args)[0] sub.url = subdata.new_url _outcome = write(conf) move = 'Feed moved to %s. ' % subdata.new_url msg = move + 'Successfully update config file' if _outcome.success \ else move + 'Failed to update config file. ' + \ 'Check permissions or update manually.' return Outcome(_outcome.success, msg)
def download_img_file(url, sub_dir, settings): '''Download an image file''' try: r = requests.get(url, timeout=60) except requests.exceptions.RequestException: return Outcome(False, 'Download of %s failed' % url) content_type = r.headers['content-type'].lower() mime_dic = {'image/bmp': '.bmp', 'image/gif': '.gif', 'image/jpeg': '.jpg', 'image/jpg': '.jpg', 'image/png': '.png', 'image/webp': '.webp'} extension = mime_dic.get(content_type, None) if extension is None: return Outcome(False, 'Download of image %s failed. Unknown MIME type.' % url) else: file_path = os.path.join(sub_dir, 'cover' + extension) with open(file_path, 'wb') as f: f.write(r.content) return Outcome(True, '')
def flush(self): '''Flush if we exceed threshold; otherwise save the buffer''' if not self.buffer: self.outcome = Outcome(None, 'Buffer was empty') return if len(self.buffer) < self.capacity: self.outcome = Outcome(None, 'Buffer no sufficiently full') self.save() return body = str() for record in self.buffer: body = body + self.format(record) + "\r\n" msg = MIMEText(body.encode('utf-8'), _charset="utf-8") msg['From'] = self.email.fromaddr.text msg['To'] = self.email.toaddr.text msg['Subject'] = Header("POCA log") if self.email.starttls == 'yes': try: smtp = smtplib.SMTP(self.email.host.text, 587, timeout=10) ehlo = smtp.ehlo() except (ConnectionRefusedError, socket.gaierror, socket.timeout) \ as error: self.outcome = Outcome(False, str(error)) self.save() return smtp.starttls() try: smtp.login(self.email.fromaddr.text, self.email.password.text) except smtplib.SMTPAuthenticationError as error: self.outcome = Outcome(False, str(error)) self.save() return else: try: smtp = smtplib.SMTP(self.email.host.text, 25, timeout=10) ehlo = smtp.ehlo() except (ConnectionRefusedError, socket.gaierror, socket.timeout) \ as error: self.outcome = Outcome(False, str(error)) self.save() return try: smtp.sendmail(self.email.fromaddr.text, [self.email.toaddr.text], msg.as_string()) self.outcome = Outcome(True, "Succesfully sent email") except (smtplib.SMTPException, socket.timeout) as error: self.outcome = Outcome(False, str(error)) self.save() return smtp.quit() self.buffer = [] self.state_jar.buffer = self.buffer self.state_jar.save()
def write_config_file(config_file_path): '''Writes default config xml to config file''' print("No config file found. Making one at %s." % config_file_path) default_base_dir = path.expanduser(path.join('~', 'poca')) query = input("Please enter the full path for placing media files.\n" "Press Enter to use default (%s): " % default_base_dir) template_file = StringIO(TEMPLATE) config_xml = objectify.parse(template_file) config_root = config_xml.getroot() config_root.settings.base_dir = query if query else default_base_dir config_xml_str = pretty_print(config_xml) try: config_file = open(config_file_path, mode='wt', encoding='utf-8') config_file.write(config_xml_str) config_file.close() msg = ("Default config succesfully written to %s.\n" "Please edit or run 'poca-subscribe' to add subscriptions." % config_file_path) return Outcome(True, msg) except IOError as e: msg = "Failed writing config to %s.\nError: %s" % (config_file_path, str(e)) return Outcome(False, msg)
def __init__(self, sub, feed, combo, del_lst, sub_dir): self.outcome = Outcome(True, 'Default true') self.lst = combo.lst self.lst = list(filter(lambda x: x not in del_lst, self.lst)) self.lst = list(filter(lambda x: combo.dic[x]['valid'], self.lst)) if hasattr(sub, 'filters'): self.apply_filters(sub, combo) if hasattr(sub, 'max_number'): self.limit(sub) self.dic = {uid: entryinfo.expand(combo.dic[uid], sub, sub_dir) for uid in self.lst} filenames = [self.dic[uid]['poca_filename'] for uid in self.lst] for uid in self.lst: count = filenames.count(self.dic[uid]['poca_filename']) if count > 1: self.dic[uid]['unique_filename'] = False else: self.dic[uid]['unique_filename'] = True self.feed_etag = feed.etag self.feed_modified = feed.modified self.feed_image = feed.image
def merge(user_el, new_el, default_el, errors=[]): '''Updating one lxml objectify elements with another (with primitive validation)''' for child in user_el.iterchildren(): new_child = new_el.find(child.tag) default_child = default_el.find(child.tag) if default_child is None: new_el.append(child) continue if isinstance(child, objectify.ObjectifiedDataElement): right_type = type(child) == type(default_child) valid = child.text in default_child.attrib.values() \ if default_child.attrib else True if all((right_type, valid)): new_el.replace(new_child, child) else: errors.append(Outcome(False, '%s: %s. Value not valid' % (child.tag, child.text))) elif isinstance(child, objectify.ObjectifiedElement): merge(child, new_child, default_child, errors=errors) return errors
def download_file(url, file_path, settings): '''Download function with block time outs''' my_thread = current_thread() headers = requests.utils.default_headers() if settings.useragent.text: useragent = {'User-Agent': settings.useragent.text} headers.update(useragent) if getattr(my_thread, "kill", False): return Outcome(None, 'Download cancelled by user') try: r = requests.get(url, stream=True, timeout=60, headers=headers) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: return Outcome(False, 'Download of %s failed' % url) except requests.exceptions.Timeout: return Outcome(False, 'Download of %s timed out' % url) if r.status_code >= 400: return Outcome(False, 'Download of %s failed' % url) with open(file_path, 'wb') as f: try: for chunk in r.iter_content(chunk_size=1024): if getattr(my_thread, "kill", False): r.close() _outcome = delete_file(f.name) return Outcome(None, 'Download cancelled by user') if chunk: f.write(chunk) except requests.exceptions.ConnectionError as e: r.close() _outcome = delete_file(f.name) return Outcome(False, 'Download of %s broke off' % url) except requests.exceptions.Timeout: r.close() _outcome = delete_file(f.name) return Outcome(False, 'Download of %s timed out' % url) r.close() return Outcome(True, '')
def tag_audio_file(settings, sub, jar, entry): '''Metdata tagging using mutagen''' id3v1_dic = {'yes': 0, 'no': 2} id3v1 = id3v1_dic[settings.id3removev1.text] id3v2 = int(settings.id3v2version) tracks = sub.find('./track_numbering') tracks = tracks.text if tracks else 'no' frames = sub.xpath('./metadata/*') invalid_keys = [] if not frames and tracks == 'no': return Outcome(True, 'Tagging skipped') # get access to metadata try: audio = mutagen.File(entry['poca_abspath']) except mutagen.MutagenError: return Outcome(False, '%s not found or invalid file type for tagging' % entry['poca_abspath']) except mutagen.mp3.HeaderNotFoundError: return Outcome(False, '%s bad mp3' % entry['poca_abspath']) if audio is None: return Outcome(False, '%s is invalid file type for tagging' % entry['poca_abspath']) if audio.tags is None: audio.add_tags() # easify if isinstance(audio, mutagen.mp3.MP3): if id3v2 == 3: audio.tags.update_to_v23() elif id3v2 == 4: audio.tags.update_to_v24() audio.save(v1=id3v1, v2_version=id3v2) audio = mutagen.File(entry['poca_abspath'], easy=True) if isinstance(audio, mutagen.mp4.MP4): audio = mutagen.File(entry['poca_abspath'], easy=True) # validate the fields and file type audio_type = type(audio) outcome, overrides, invalid_keys = validate_keys(audio_type, frames) if outcome.success is False: return outcome # run overrides for override in overrides: audio[override[0]] = override[1] if tracks == 'yes' or (tracks == 'if missing' and 'tracknumber' not in audio): track_no = jar.track_no if hasattr(jar, 'track_no') else 0 track_no += 1 jar.track_no = track_no jar.save() if isinstance(audio, (mutagen.mp3.EasyMP3, mutagen.oggvorbis.OggVorbis, mutagen.oggopus.OggOpus, mutagen.flac.FLAC)): audio['tracknumber'] = str(track_no) # else? # save and finish if isinstance(audio, mutagen.mp3.EasyMP3): audio.save(v1=id3v1, v2_version=id3v2) else: audio.save() if not invalid_keys: return Outcome(True, 'Metadata successfully updated') else: return Outcome(False, '%s is set to add invalid tags: %s' % (sub.title.text, ', '.join(invalid_keys)))
def tag_audio_file(settings, sub, jar, entry): '''Metadata tagging using mutagen''' # id3 settings id3v1 = id3v1_dic[settings.id3removev1.text] id3v2 = int(settings.id3v2version) id3encoding = encodings[id3v2] # overrides frames = sub.xpath('./metadata/*') overrides = [(override.tag, override.text) for override in frames] key_errors = {} # track numbering tracks = sub.find('./track_numbering') tracks = tracks.text if tracks else 'no' if not overrides and tracks == 'no': return Outcome(True, 'Tagging skipped') # get 'easy' access to metadata try: audio = mutagen.File(entry['poca_abspath'], easy=True) except mutagen.MutagenError: return Outcome( False, '%s not found or invalid file type for tagging' % entry['poca_abspath']) except mutagen.mp3.HeaderNotFoundError: return Outcome(False, '%s is a bad mp3' % entry['poca_abspath']) if audio is None: return Outcome( False, '%s is invalid file type for tagging' % entry['poca_abspath']) # add_tags is undocumented for easy but seems to work if audio.tags is None: audio.add_tags() # tracks if tracks == 'yes' or (tracks == 'if missing' and 'tracknumber' not in audio): track_no = jar.track_no if hasattr(jar, 'track_no') else 0 track_no += 1 overrides.append(('tracknumber', str(track_no))) jar.track_no = track_no jar.save() # run overrides and save while overrides: tag, text = overrides.pop() if not text and tag in audio: _text = audio.pop(tag) continue try: audio[tag] = text except (easyid3.EasyID3KeyError, easymp4.EasyMP4KeyError, \ ValueError) as e: key_errors[tag] = text audio.save() # ONLY for ID3 if isinstance(audio, mutagen.mp3.EasyMP3): audio = mutagen.File(entry['poca_abspath'], easy=False) if 'comment' in key_errors: audio.tags.delall('COMM') comm_txt = key_errors.pop('comment') if comm_txt: comm = id3.COMM(encoding=id3encoding, lang='eng', \ desc='desc', text=comm_txt) audio.tags.add(comm) if 'chapters' in key_errors: _toc = key_errors.pop('chapters') audio.tags.delall('CTOC') audio.tags.delall('CHAP') if id3v2 == 3: audio.tags.update_to_v23() elif id3v2 == 4: audio.tags.update_to_v24() audio.save(v1=id3v1, v2_version=id3v2) # invalid keys invalid_keys = list(key_errors.keys()) if not invalid_keys: return Outcome(True, 'Metadata successfully updated') else: return Outcome( False, '%s is set to add invalid tags: %s' % (sub.title.text, ', '.join(invalid_keys)))
def verify_file(entry): '''Check to see if recorded file exists or has been removed''' isfile = os.path.isfile(entry['poca_abspath']) return Outcome(isfile, entry['poca_abspath'] + ' exists: ' + str(isfile))