def ConverterHandler(args): """ Function that handles the Converter in a loop """ # Load Chapters! chapters = helper.getChapters() # Start conversion loop! for chapter in chapters.iterator(): # Verify if chapter has been downloaded already if not helper.verifyDownload(chapter): logging.debug("Manga %s has not been downloaded!", chapter.title) else: # Spawn an Converter Object & get basic data from database & config current_conversation = Converter() current_conversation.data_collector(chapter) # Check if Download loop & Download task is selected if not args.start: current_conversation.cbz_creator() current_conversation.eb_creator() else: # Only start run if chapter is younger than 24h if helper.checkTime(current_conversation.chapterdate): current_conversation.cbz_creator() current_conversation.eb_creator() else: logging.debug( "%s is older than 24h, will not be processed by daemon.", current_conversation.mangatitle)
def __init__(self): # Python 3 is required! if sys.version_info[0] < 3: sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n") sys.exit(1) # Get args right at the start self.args = None if not self.args: self.read_arguments() # Load config right at the start self.config = None if not self.config: self.config = Config.load_config() logging.debug("Loaded Config:") logging.debug(self.config) # Check if Database exists, else create if not os.path.isfile(self.config["Database"]): helper.createFolder(self.config["SaveLocation"]) helper.createDB() # Check weather there are some database migrations mversion = helper.getMigrationVersion() + ".py" if self.config["DisableMigrations"] == "True": logging.debug("Migrations disabled! Current version: %s ", mversion) else: if mversion in os.listdir(os.getcwd() + "/migrations"): logging.debug("No migrations required! Current version: %s ", mversion) else: migrator.migrate()
def downloader(self, url, counter, parser): # Check if we have the Download folder helper.createFolder(self.downloadfolder) imagepath = self.downloadfolder + "/" + str( "{0:0=3d}".format(counter)) + ".png" tempdl = self.downloadfolder + "/" + str( "{0:0=3d}".format(counter)) + ".tmp" # Download the image! f = open(tempdl, 'wb') f.write(requests.get(parser(url)).content) f.close() # If everything is alright, write image to final name os.rename(tempdl, imagepath) # Cleanse image, remove footer # # I have borrowed this code from the kmanga project. # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 # Thanks a lot to Alberto Planas for coming up with it! # if self.origin == "mangafox.me" or self.origin == "mangafox.la" or self.origin == "fanfox.net": logging.debug("Cleaning Mangafox Footer") img = Image.open(imagepath) _img = ImageOps.invert(img.convert(mode='L')) _img = _img.point(lambda x: x and 255) _img = _img.filter(ImageFilter.MinFilter(size=3)) _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) _img = _img.point(lambda x: (x >= 48) and x) cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img cleaned.save(imagepath)
def directConverter(chapterids=[]): """ Function that handles direct calls of the Converter """ logging.debug("Following Chapters are directly converted:") logging.debug(chapterids) chapters = helper.getChaptersFromID(chapterids) if not chapters: logging.error("No Chapters found with said ID!") else: # Start conversion loop! for chapter in chapters: # Verify if chapter has been downloaded already if not helper.verifyDownload(chapter): logging.info("Manga %s has not been downloaded!", chapter[2]) else: # Spawn an Converter Object & get basic data from database & config current_conversation = Converter() current_conversation.data_collector(chapter) if os.path.exists(current_conversation.cbzlocation): logging.info("Manga %s converted to CBZ already!", current_conversation.mangatitle) else: current_conversation.cbz_creator() # Start conversion to Ebook format! if os.path.exists(current_conversation.eblocation): logging.info("Manga %s converted to Ebook already!", current_conversation.mangatitle) else: current_conversation.eb_creator()
def data_collector(self, chapter): """ Method that gathers data required for this class """ # Load config right at the start config = None if not config: config = Config.load_config() # Load configs required here self.saveloc = config["SaveLocation"] self.ebformat = config["EbookFormat"] self.smtpserver = config["SMTPServer"] self.serverport = config["ServerPort"] self.emailadress = config["EmailAddress"] self.password = config["EmailAddressPw"] self.starttls = config["ServerStartSSL"] # get relevant data of this Manga self.mangatitle = chapter.title self.chapterdate = chapter.date self.mangaid = int(chapter.chapterid) self.issent = int(chapter.issent) self.manganame = chapter.manganame # check if mangatitle or manganame contains ":" characters that OS can't handle as folders self.mangatitle = helper.sanetizeName(self.mangatitle) self.manganame = helper.sanetizeName(self.manganame) self.eblocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" + self.mangatitle + "." + self.ebformat.lower()) # Initialize emtpy Users table self.users = []
def data_collector(self, chapter): # Load config right at the start config = None if not config: config = Config.load_config() # Load configs required here self.database = config["Database"] self.saveloc = config["SaveLocation"] # get relevant data of this Chapter self.mangastarturl = chapter.url self.mangapages = chapter.pages self.mangatitle = chapter.title self.manganame = chapter.manganame self.chapterdate = chapter.date # check if mangatitle or manganame contains ":" characters that OS can't handle as folders self.mangatitle = helper.sanetizeName(self.mangatitle) self.manganame = helper.sanetizeName(self.manganame) # Define Download location self.downloadfolder = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/images") # get Origin of manga (Which mangawebsite) self.origin = helper.getSourceURL(self.mangastarturl) # Initiate URL list self.imageurls = []
def data_collector(self, chapter): """ Method that collects data""" # Load config right at the start config = None if not config: config = Config.load_config() # Load configs required here self.saveloc = config["SaveLocation"] self.ebformat = config["EbookFormat"] self.ebprofile = config["EbookProfile"] # get relevant data of this Manga self.mangatitle = chapter.title self.manganame = chapter.manganame self.chapterdate = chapter.date # check if mangatitle or manganame contains ":" characters that OS can't handle as folders self.mangatitle = helper.sanetizeName(self.mangatitle) self.manganame = helper.sanetizeName(self.manganame) # create folder variables self.imagefolder = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/images/") self.eblocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" + self.mangatitle + "." + self.ebformat.lower()) self.cbzlocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" + self.mangatitle + ".cbz")
def directSender(chapterids=[]): """ Function that handles the coordination of directly sending ebooks """ logging.debug("Following Chapters are directly sent:") logging.debug(chapterids) chapters = helper.getChaptersFromID(chapterids) # Load Users users = helper.getUsers() # Debug Users: logging.debug("Userlist:") for i in users: logging.debug(i.name) if not chapters: logging.error("No Chapters found with said ID!") else: # Start conversion loop! for chapter in chapters: # Initiate Sender class and fill it with data current_sender = Sender() current_sender.data_collector(chapter) current_sender.users = users # Check if ebook has been converted yet, else skip if not os.path.exists(current_sender.eblocation): logging.debug("Manga %s has not been converted yet.", current_sender.mangatitle) else: logging.info("Sending %s...", current_sender.mangatitle) current_sender.send_eb()
def add_filter(self): if len(self.args.filter) > 0: filter_value = self.args.filter[0] logging.debug("Entered filter: %s", filter_value) helper.writeFilter(filter_value) else: logging.error("You need to enter a filter value!")
def SenderHandler(args): """ Function that handles the sending of ebooks when a loop is called """ # Get all Chapters chapters = helper.getChapters() # Load Users users = helper.getUsers() # Debug Users: logging.debug("Userlist:") for i in users: logging.debug(i.name) # Start conversion loop! for chapter in chapters.iterator(): # Initiate Sender class and fill it with data current_sender = Sender() current_sender.data_collector(chapter) current_sender.users = users # Check if ebook has been converted yet, else skip if not os.path.exists(current_sender.eblocation): logging.debug("Manga %s has not been converted yet.", current_sender.mangatitle) else: # Check if Chapter has been sent already if current_sender.issent != 0: logging.debug("%s has been sent already!", current_sender.mangatitle) else: # Check if Sender loop or Sender task is selected if not args.start: logging.info("Sending %s...", current_sender.mangatitle) current_sender.send_eb() else: # Only start run if chapter is younger than 24h if helper.checkTime(current_sender.chapterdate): logging.info("Sending %s...", current_sender.mangatitle) current_sender.send_eb() else: logging.debug( "%s is older than 24h, will not be processed by daemon.", current_sender.mangatitle)
def load_from_feed(self, entry, parent_feed): self.chapter_link = entry.link # Check if link is already in DB to make sure only data gets downloaded that is not yet downloaded logging.debug("Checking if chapter is already saved...") db.connection() self.duplicated = Chapter.select().where( Chapter.url == self.chapter_link) if self.duplicated.exists(): logging.debug("Manga is already in Database! Skipping...") else: # Getting specific manga data logging.debug("Fetching Data from Weblink") mangadata = helper.getMangaData(self.chapter_link, entry) logging.debug("Finished Collecting Chapter Data!") self.manga_name = mangadata[0] self.title = mangadata[3] self.chapter = mangadata[2] self.chapter_name = mangadata[4] self.chapter_pages = mangadata[1] self.chapter_pubDate = mangadata[5] self.parent_feed = parent_feed # Set some defaul values self.ispulled = 0 self.isconverted = 0 self.issent = 0
def downloader(args): """ the downloader function """ # Make the query chapters = helper.getChapters() if args.start: logging.debug("The loop will only consider Chapters younger than 24h!") # Start Download loop! for chapter in chapters.iterator(): # Initialize Downloader class & load basic params current_chapter = Downloader() current_chapter.data_collector(chapter) # Check if the old DL location is being used and fix it! oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) newlocation = str(current_chapter.saveloc + current_chapter.manganame) if os.path.isdir(oldlocation): logging.info("Moving %s from old DL location to new one...", current_chapter.mangatitle) helper.createFolder(newlocation) move(oldlocation, newlocation) # Check if chapter needs to be downloaded if helper.verifyDownload(chapter): logging.debug("Manga %s downloaded already!", current_chapter.mangatitle) else: # Check if Download loop & Download task is selected if not args.start: current_chapter.data_processor() else: # Only start run if chapter is younger than 24h if helper.checkTime(current_chapter.chapterdate): current_chapter.data_processor() else: logging.debug("%s is older than 24h, will not be processed by daemon.", current_chapter.mangatitle)
def directDownloader(chapterids=[]): logging.debug("Following Chapters are directly converted:") logging.debug(chapterids) chapters = helper.getChaptersFromID(chapterids) # Load Users users = helper.getUsers() # Debug Users: logging.debug("Userlist:") logging.debug(users) if not chapters: logging.error("No Chapters found with said ID!") else: # Start conversion loop! for chapter in chapters: # Initialize Downloader class & load basic params current_chapter = Downloader() current_chapter.data_collector(chapter) # Check if the old DL location is being used and fix it! oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) newlocation = str(current_chapter.saveloc + current_chapter.manganame) if os.path.isdir(oldlocation): logging.info("Moving %s from old DL location to new one..." % current_chapter.mangatitle) helper.createFolder(newlocation) move(oldlocation, newlocation) # Check if chapter needs to be downloaded if helper.verifyDownload(chapter): logging.info("Manga %s downloaded already!" % current_chapter.mangatitle) else: current_chapter.data_processor()
def __init__(self): # Python 3 is required! if sys.version_info[0] < 3: sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n") sys.exit(1) # Get args right at the start self.args = None if not self.args: self.read_arguments() # Load config right at the start self.config = None if not self.config: self.config = Config.load_config() logging.debug("Loaded Config:") logging.debug(self.config) # Check if Database exists, else create if not os.path.isfile(self.config["Database"]): helper.createFolder(self.config["SaveLocation"]) helper.createDB()
def create_db(self): helper.createDB()
def add_user(self): helper.createUser()
def list_users(self): helper.printUsers()
def list_manga(self): helper.printManga(self.args)
def list_chapters(self): helper.printChapters()
def list_all_chapters(self): helper.printChaptersAll()
def setUp(self): self.user = "******" self.help = Helper()
def delete_feed(self): logging.debug("Entered USERID: %s", self.args.delete_feed) helper.deleteFeed(self.args.delete_feed)
def delete_chapter(self): logging.debug("Entered USERID: %s", self.args.delete_chapter) helper.deleteChapter(self.args.delete_chapter)
def switch_user_status(self): logging.debug("Entered USERID: %s", self.args.switch_send) helper.switchUserSend(self.args.switch_send)
def save_feed_to_db(self): logging.debug("Entered URL: %s", self.args.add_feed) if validators.url(self.args.add_feed): helper.writeFeed(self.args.add_feed) else: logging.error("You need to enter an URL!")
def send_eb(self): """ Method that sends data to the user! """ # Iterate through user for user in self.users: kindle_mail = user.kindle_mail shouldsend = user.sendtokindle user_mail = user.email # Check if user wants Mails if shouldsend == 1: logging.debug("Compiling Email for %s", user.name) # Compile Email msg = MIMEMultipart() msg['Subject'] = 'Ebook Delivery of %s' % self.mangatitle msg['Date'] = formatdate(localtime=True) msg['From'] = self.emailadress msg['To'] = kindle_mail msg['Message-ID'] = make_msgid() text = "Automatic Ebook delivery by m2em." msg.attach(MIMEText(text)) # Add Ebook as attachment ebfile = open(self.eblocation, 'rb') attachment = MIMEBase('application', 'octet-stream', name=os.path.basename(self.eblocation)) attachment.set_payload(ebfile.read()) ebfile.close() encoders.encode_base64(attachment) attachment.add_header('Content-Disposition', 'attachment', filename=os.path.basename(self.eblocation)) msg.attach(attachment) # Convert message to string sio = StringIO() gen = Generator(sio, mangle_from_=False) gen.flatten(msg) msg = sio.getvalue() # Send Email Off! # Debug Server Data logging.debug("Server: %s", self.smtpserver) logging.debug("Port: %s", self.serverport) try: server = smtplib.SMTP(self.smtpserver, self.serverport,) if self.starttls: server.starttls() server.ehlo() server.login(self.emailadress, self.password) #server.sendmail(emailadress, kindle_mail, msg.as_string()) server.sendmail(self.emailadress, kindle_mail, msg) server.close() logging.debug("Sent Ebook email to %s ", kindle_mail) self.send_confirmation(user_mail) except smtplib.SMTPException as fail: logging.debug("Could not send email! %s", fail) # Set Email as Sent helper.setIsSent(self.mangaid) logging.info("Sent %s to all requested users.", self.mangatitle)
class TestGitlabClient(unittest.TestCase): """ Test class for GitlabClient""" def setUp(self): self.user = "******" self.help = Helper() def test_get_entry(self): print "Get entry" userlist = json.loads(json.dumps([ { "id": 1, "username": "******", }, { "id": 2, "username": "******", } ])) with mock.patch('__builtin__.raw_input', return_value='0'): assert self.help.get_entry(userlist, 'username') == 1 with mock.patch('__builtin__.raw_input', return_value='1'): assert self.help.get_entry(userlist, 'username') == 2 with mock.patch('__builtin__.raw_input', return_value='3'): self.assertRaises(ValueError) userlist = json.loads(json.dumps([ { "id": 1, "username": "******", } ])) self.assertEquals( self.help.get_entry(userlist, 'username'), 1) def test_get_entry_project_id(self): print "Get entry project id" projects = json.loads(json.dumps([ { "id": 4, "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", }, { "id": 6, "http_url_to_repo": "http://example.com/brightbox/puppet.git", }, { "id": 8, "http_url_to_repo": "http://example.com/birghtbox/puppet", } ])) with mock.patch('__builtin__.raw_input', return_value='0'): assert self.help.get_entry(projects, 'http_url_to_repo') == 4 with mock.patch('__builtin__.raw_input', return_value='1'): assert self.help.get_entry(projects, 'http_url_to_repo') == 6 with mock.patch('__builtin__.raw_input', return_value='3'): self.assertRaises(ValueError) reponame = "http://example.com/diaspora/diaspora-client.git" filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo') self.assertEquals(len(filtered), 1) self.assertEquals(filtered[0]['http_url_to_repo'], reponame) projects = json.loads(json.dumps([ { "id": 4, "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", }, { "id": 6, "http_url_to_repo": "http://example.com/diaspora/diaspora", }, { "id": 8, "http_url_to_repo": "http://example.com/diaspora/diaspora-client", }, { "id": 10, "http_url_to_repo": "http://example.com/diaspora/diaspora.git", } ])) reponame = "http://example.com/diaspora/diaspora" filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo') self.assertEquals(len(filtered), 4) self.assertEquals(filtered[0]['http_url_to_repo'], "http://example.com/diaspora/diaspora-client.git") reponame = "http://example.com/diaspora/diaspora.git" filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo') self.assertEquals(len(filtered), 1) self.assertEquals(filtered[0]['http_url_to_repo'], reponame) reponame = "http://example.com/diaspora/diasporasdfadf.git" filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo') self.assertEquals(len(filtered), 0) def test_apicall(self): print "Test apicall" text = apicall(test_func, "fsk", "lks", "slkdfj", test="lkjsf") self.assertEquals(text, "fsk lks slkdfj lkjsf") def test_get_state_event(self): print "Test get state event" with mock.patch('__builtin__.raw_input', return_value='0'): assert self.help.get_state_event() == "close" with mock.patch('__builtin__.raw_input', return_value='1'): assert self.help.get_state_event() == "reopen" with mock.patch('__builtin__.raw_input', return_value='2'): assert self.help.get_state_event() == "merge" with mock.patch('__builtin__.raw_input', return_value='3'): self.assertRaises(ValueError) def test_show_info(self): # not really a test maybe a print test later print "Test show infos dummy" entry = { "id": 4, "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", } self.help.show_infos(entry, "List", "id", "http_url_to_repo")
def read_arguments(self): """ function that reads all arguments """ # Get user Input parser = argparse.ArgumentParser(description='Manga to eManga - m2em') parser.add_argument( "-af", "--add-feed", help= "Add RSS Feed of Manga. Only Mangastream & MangaFox are supported") parser.add_argument("-au", "--add-user", help="Adds new user", action="store_true") parser.add_argument( "-lm", "--list-manga", help= "Lists Manga saved in database. If a Manga is passed, lists chapters to said Manga", nargs="?", const='all') parser.add_argument("-lc", "--list-chapters", help="Lists the last 10 Chapters", action="store_true") parser.add_argument("-Lc", "--list-chapters-all", help="Lists all Chapters", action="store_true") parser.add_argument("-lf", "--list-feeds", help="Lists all feeds", action="store_true") parser.add_argument("-lu", "--list-users", help="Lists all Users", action="store_true") parser.add_argument("-cd", "--create-db", help="Creates DB. Uses Configfile for Naming", action="store_true") parser.add_argument("-s", "--start", help="Starts one loop", action="store_true") parser.add_argument( "--send", help= "Sends Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs='*', ) parser.add_argument( "--convert", help= "Converts Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs='*', ) parser.add_argument( "--download", help= "Downloads Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs='*', ) parser.add_argument( "-p", "--process", help= "Processes chapter(s) by chapter ID, Download, convert, send. Multiple IDs can be given", default=[], nargs='*', ) parser.add_argument( "-a", "--action", help= "Start action. Options are: rssparser (collecting feed data), downloader, converter or sender " ) parser.add_argument( "-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") parser.add_argument("-dc", "--delete-chapter", help="Pass ID of Chapter. Deletes said Chapter") parser.add_argument("-du", "--delete-user", help="Pass ID of User. Deletes said User") parser.add_argument("-df", "--delete-feed", help="Pass ID of Feed. Deletes said Feed") parser.add_argument("--daemon", help="Run as daemon", action="store_true") parser.add_argument("-d", "--debug", help="Debug Mode", action="store_true") parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__) self.args = parser.parse_args() # Logging if self.args.debug: outputlevel = "debug" else: outputlevel = "info" helper.initialize_logger("log/", outputlevel) # Check if Arguments are passed or not. At least one is required if self.args.action is None \ and self.args.add_feed is None \ and self.args.delete_chapter is None \ and self.args.delete_feed is None \ and self.args.delete_user is None \ and self.args.switch_send is None \ and self.args.add_user is False \ and self.args.list_manga is None \ and not any([self.args.add_user, self.args.create_db, self.args.daemon, self.args.list_chapters, self.args.list_chapters_all, self.args.list_feeds, self.args.list_users, self.args.download, self.args.convert, self.args.send, self.args.process, self.args.start,]): logging.error("At least one argument is required!") logging.debug("Passed arguments: \n %s", self.args)
def list_feeds(self): helper.printFeeds()
def filter_list(self): helper.printFilters()