def ConverterHandler(args):
""" Function that handles the Converter in a loop """
# Load Chapters!
chapters = helper.getChapters()
# Start conversion loop!
for chapter in chapters.iterator():
# Verify if chapter has been downloaded already
if not helper.verifyDownload(chapter):
logging.debug("Manga %s has not been downloaded!", chapter.title)
else:
# Spawn an Converter Object & get basic data from database & config
current_conversation = Converter()
current_conversation.data_collector(chapter)
# Check if Download loop & Download task is selected
if not args.start:
current_conversation.cbz_creator()
current_conversation.eb_creator()
else:
# Only start run if chapter is younger than 24h
if helper.checkTime(current_conversation.chapterdate):
current_conversation.cbz_creator()
current_conversation.eb_creator()
else:
logging.debug(
"%s is older than 24h, will not be processed by daemon.",
current_conversation.mangatitle)
def __init__(self):
# Python 3 is required!
if sys.version_info[0] < 3:
sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
sys.exit(1)
# Get args right at the start
self.args = None
if not self.args:
self.read_arguments()
# Load config right at the start
self.config = None
if not self.config:
self.config = Config.load_config()
logging.debug("Loaded Config:")
logging.debug(self.config)
# Check if Database exists, else create
if not os.path.isfile(self.config["Database"]):
helper.createFolder(self.config["SaveLocation"])
helper.createDB()
# Check weather there are some database migrations
mversion = helper.getMigrationVersion() + ".py"
if self.config["DisableMigrations"] == "True":
logging.debug("Migrations disabled! Current version: %s ",
mversion)
else:
if mversion in os.listdir(os.getcwd() + "/migrations"):
logging.debug("No migrations required! Current version: %s ",
mversion)
else:
migrator.migrate()
def downloader(self, url, counter, parser):
# Check if we have the Download folder
helper.createFolder(self.downloadfolder)
imagepath = self.downloadfolder + "/" + str(
"{0:0=3d}".format(counter)) + ".png"
tempdl = self.downloadfolder + "/" + str(
"{0:0=3d}".format(counter)) + ".tmp"
# Download the image!
f = open(tempdl, 'wb')
f.write(requests.get(parser(url)).content)
f.close()
# If everything is alright, write image to final name
os.rename(tempdl, imagepath)
# Cleanse image, remove footer
#
# I have borrowed this code from the kmanga project.
# https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416
# Thanks a lot to Alberto Planas for coming up with it!
#
if self.origin == "mangafox.me" or self.origin == "mangafox.la" or self.origin == "fanfox.net":
logging.debug("Cleaning Mangafox Footer")
img = Image.open(imagepath)
_img = ImageOps.invert(img.convert(mode='L'))
_img = _img.point(lambda x: x and 255)
_img = _img.filter(ImageFilter.MinFilter(size=3))
_img = _img.filter(ImageFilter.GaussianBlur(radius=5))
_img = _img.point(lambda x: (x >= 48) and x)
cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img
cleaned.save(imagepath)
def directConverter(chapterids=[]):
""" Function that handles direct calls of the Converter """
logging.debug("Following Chapters are directly converted:")
logging.debug(chapterids)
chapters = helper.getChaptersFromID(chapterids)
if not chapters:
logging.error("No Chapters found with said ID!")
else:
# Start conversion loop!
for chapter in chapters:
# Verify if chapter has been downloaded already
if not helper.verifyDownload(chapter):
logging.info("Manga %s has not been downloaded!", chapter[2])
else:
# Spawn an Converter Object & get basic data from database & config
current_conversation = Converter()
current_conversation.data_collector(chapter)
if os.path.exists(current_conversation.cbzlocation):
logging.info("Manga %s converted to CBZ already!",
current_conversation.mangatitle)
else:
current_conversation.cbz_creator()
# Start conversion to Ebook format!
if os.path.exists(current_conversation.eblocation):
logging.info("Manga %s converted to Ebook already!",
current_conversation.mangatitle)
else:
current_conversation.eb_creator()
def data_collector(self, chapter):
""" Method that gathers data required for this class """
# Load config right at the start
config = None
if not config:
config = Config.load_config()
# Load configs required here
self.saveloc = config["SaveLocation"]
self.ebformat = config["EbookFormat"]
self.smtpserver = config["SMTPServer"]
self.serverport = config["ServerPort"]
self.emailadress = config["EmailAddress"]
self.password = config["EmailAddressPw"]
self.starttls = config["ServerStartSSL"]
# get relevant data of this Manga
self.mangatitle = chapter.title
self.chapterdate = chapter.date
self.mangaid = int(chapter.chapterid)
self.issent = int(chapter.issent)
self.manganame = chapter.manganame
# check if mangatitle or manganame contains ":" characters that OS can't handle as folders
self.mangatitle = helper.sanetizeName(self.mangatitle)
self.manganame = helper.sanetizeName(self.manganame)
self.eblocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" +
self.mangatitle + "." + self.ebformat.lower())
# Initialize emtpy Users table
self.users = []
def data_collector(self, chapter):
# Load config right at the start
config = None
if not config:
config = Config.load_config()
# Load configs required here
self.database = config["Database"]
self.saveloc = config["SaveLocation"]
# get relevant data of this Chapter
self.mangastarturl = chapter.url
self.mangapages = chapter.pages
self.mangatitle = chapter.title
self.manganame = chapter.manganame
self.chapterdate = chapter.date
# check if mangatitle or manganame contains ":" characters that OS can't handle as folders
self.mangatitle = helper.sanetizeName(self.mangatitle)
self.manganame = helper.sanetizeName(self.manganame)
# Define Download location
self.downloadfolder = str(self.saveloc + self.manganame + "/" +
self.mangatitle + "/images")
# get Origin of manga (Which mangawebsite)
self.origin = helper.getSourceURL(self.mangastarturl)
# Initiate URL list
self.imageurls = []
def data_collector(self, chapter):
""" Method that collects data"""
# Load config right at the start
config = None
if not config:
config = Config.load_config()
# Load configs required here
self.saveloc = config["SaveLocation"]
self.ebformat = config["EbookFormat"]
self.ebprofile = config["EbookProfile"]
# get relevant data of this Manga
self.mangatitle = chapter.title
self.manganame = chapter.manganame
self.chapterdate = chapter.date
# check if mangatitle or manganame contains ":" characters that OS can't handle as folders
self.mangatitle = helper.sanetizeName(self.mangatitle)
self.manganame = helper.sanetizeName(self.manganame)
# create folder variables
self.imagefolder = str(self.saveloc + self.manganame + "/" +
self.mangatitle + "/images/")
self.eblocation = str(self.saveloc + self.manganame + "/" +
self.mangatitle + "/" + self.mangatitle + "." +
self.ebformat.lower())
self.cbzlocation = str(self.saveloc + self.manganame + "/" +
self.mangatitle + "/" + self.mangatitle +
".cbz")
def directSender(chapterids=[]):
""" Function that handles the coordination of directly sending ebooks """
logging.debug("Following Chapters are directly sent:")
logging.debug(chapterids)
chapters = helper.getChaptersFromID(chapterids)
# Load Users
users = helper.getUsers()
# Debug Users:
logging.debug("Userlist:")
for i in users:
logging.debug(i.name)
if not chapters:
logging.error("No Chapters found with said ID!")
else:
# Start conversion loop!
for chapter in chapters:
# Initiate Sender class and fill it with data
current_sender = Sender()
current_sender.data_collector(chapter)
current_sender.users = users
# Check if ebook has been converted yet, else skip
if not os.path.exists(current_sender.eblocation):
logging.debug("Manga %s has not been converted yet.",
current_sender.mangatitle)
else:
logging.info("Sending %s...", current_sender.mangatitle)
current_sender.send_eb()
def add_filter(self):
if len(self.args.filter) > 0:
filter_value = self.args.filter[0]
logging.debug("Entered filter: %s", filter_value)
helper.writeFilter(filter_value)
else:
logging.error("You need to enter a filter value!")
def SenderHandler(args):
""" Function that handles the sending of ebooks when a loop is called """
# Get all Chapters
chapters = helper.getChapters()
# Load Users
users = helper.getUsers()
# Debug Users:
logging.debug("Userlist:")
for i in users:
logging.debug(i.name)
# Start conversion loop!
for chapter in chapters.iterator():
# Initiate Sender class and fill it with data
current_sender = Sender()
current_sender.data_collector(chapter)
current_sender.users = users
# Check if ebook has been converted yet, else skip
if not os.path.exists(current_sender.eblocation):
logging.debug("Manga %s has not been converted yet.",
current_sender.mangatitle)
else:
# Check if Chapter has been sent already
if current_sender.issent != 0:
logging.debug("%s has been sent already!",
current_sender.mangatitle)
else:
# Check if Sender loop or Sender task is selected
if not args.start:
logging.info("Sending %s...", current_sender.mangatitle)
current_sender.send_eb()
else:
# Only start run if chapter is younger than 24h
if helper.checkTime(current_sender.chapterdate):
logging.info("Sending %s...",
current_sender.mangatitle)
current_sender.send_eb()
else:
logging.debug(
"%s is older than 24h, will not be processed by daemon.",
current_sender.mangatitle)
def load_from_feed(self, entry, parent_feed):
self.chapter_link = entry.link
# Check if link is already in DB to make sure only data gets downloaded that is not yet downloaded
logging.debug("Checking if chapter is already saved...")
db.connection()
self.duplicated = Chapter.select().where(
Chapter.url == self.chapter_link)
if self.duplicated.exists():
logging.debug("Manga is already in Database! Skipping...")
else:
# Getting specific manga data
logging.debug("Fetching Data from Weblink")
mangadata = helper.getMangaData(self.chapter_link, entry)
logging.debug("Finished Collecting Chapter Data!")
self.manga_name = mangadata[0]
self.title = mangadata[3]
self.chapter = mangadata[2]
self.chapter_name = mangadata[4]
self.chapter_pages = mangadata[1]
self.chapter_pubDate = mangadata[5]
self.parent_feed = parent_feed
# Set some defaul values
self.ispulled = 0
self.isconverted = 0
self.issent = 0
def downloader(args):
""" the downloader function """
# Make the query
chapters = helper.getChapters()
if args.start:
logging.debug("The loop will only consider Chapters younger than 24h!")
# Start Download loop!
for chapter in chapters.iterator():
# Initialize Downloader class & load basic params
current_chapter = Downloader()
current_chapter.data_collector(chapter)
# Check if the old DL location is being used and fix it!
oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle)
newlocation = str(current_chapter.saveloc + current_chapter.manganame)
if os.path.isdir(oldlocation):
logging.info("Moving %s from old DL location to new one...", current_chapter.mangatitle)
helper.createFolder(newlocation)
move(oldlocation, newlocation)
# Check if chapter needs to be downloaded
if helper.verifyDownload(chapter):
logging.debug("Manga %s downloaded already!", current_chapter.mangatitle)
else:
# Check if Download loop & Download task is selected
if not args.start:
current_chapter.data_processor()
else:
# Only start run if chapter is younger than 24h
if helper.checkTime(current_chapter.chapterdate):
current_chapter.data_processor()
else:
logging.debug("%s is older than 24h, will not be processed by daemon.", current_chapter.mangatitle)
def directDownloader(chapterids=[]):
logging.debug("Following Chapters are directly converted:")
logging.debug(chapterids)
chapters = helper.getChaptersFromID(chapterids)
# Load Users
users = helper.getUsers()
# Debug Users:
logging.debug("Userlist:")
logging.debug(users)
if not chapters:
logging.error("No Chapters found with said ID!")
else:
# Start conversion loop!
for chapter in chapters:
# Initialize Downloader class & load basic params
current_chapter = Downloader()
current_chapter.data_collector(chapter)
# Check if the old DL location is being used and fix it!
oldlocation = str(current_chapter.saveloc +
current_chapter.mangatitle)
newlocation = str(current_chapter.saveloc +
current_chapter.manganame)
if os.path.isdir(oldlocation):
logging.info("Moving %s from old DL location to new one..." %
current_chapter.mangatitle)
helper.createFolder(newlocation)
move(oldlocation, newlocation)
# Check if chapter needs to be downloaded
if helper.verifyDownload(chapter):
logging.info("Manga %s downloaded already!" %
current_chapter.mangatitle)
else:
current_chapter.data_processor()
def __init__(self):
# Python 3 is required!
if sys.version_info[0] < 3:
sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
sys.exit(1)
# Get args right at the start
self.args = None
if not self.args:
self.read_arguments()
# Load config right at the start
self.config = None
if not self.config:
self.config = Config.load_config()
logging.debug("Loaded Config:")
logging.debug(self.config)
# Check if Database exists, else create
if not os.path.isfile(self.config["Database"]):
helper.createFolder(self.config["SaveLocation"])
helper.createDB()
def create_db(self):
helper.createDB()
def add_user(self):
helper.createUser()
def list_users(self):
helper.printUsers()
def list_manga(self):
helper.printManga(self.args)
def list_chapters(self):
helper.printChapters()
def list_all_chapters(self):
helper.printChaptersAll()
def setUp(self):
self.user = "******"
self.help = Helper()
def delete_feed(self):
logging.debug("Entered USERID: %s", self.args.delete_feed)
helper.deleteFeed(self.args.delete_feed)
def delete_chapter(self):
logging.debug("Entered USERID: %s", self.args.delete_chapter)
helper.deleteChapter(self.args.delete_chapter)
def switch_user_status(self):
logging.debug("Entered USERID: %s", self.args.switch_send)
helper.switchUserSend(self.args.switch_send)
def save_feed_to_db(self):
logging.debug("Entered URL: %s", self.args.add_feed)
if validators.url(self.args.add_feed):
helper.writeFeed(self.args.add_feed)
else:
logging.error("You need to enter an URL!")
def send_eb(self):
""" Method that sends data to the user! """
# Iterate through user
for user in self.users:
kindle_mail = user.kindle_mail
shouldsend = user.sendtokindle
user_mail = user.email
# Check if user wants Mails
if shouldsend == 1:
logging.debug("Compiling Email for %s", user.name)
# Compile Email
msg = MIMEMultipart()
msg['Subject'] = 'Ebook Delivery of %s' % self.mangatitle
msg['Date'] = formatdate(localtime=True)
msg['From'] = self.emailadress
msg['To'] = kindle_mail
msg['Message-ID'] = make_msgid()
text = "Automatic Ebook delivery by m2em."
msg.attach(MIMEText(text))
# Add Ebook as attachment
ebfile = open(self.eblocation, 'rb')
attachment = MIMEBase('application', 'octet-stream',
name=os.path.basename(self.eblocation))
attachment.set_payload(ebfile.read())
ebfile.close()
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment',
filename=os.path.basename(self.eblocation))
msg.attach(attachment)
# Convert message to string
sio = StringIO()
gen = Generator(sio, mangle_from_=False)
gen.flatten(msg)
msg = sio.getvalue()
# Send Email Off!
# Debug Server Data
logging.debug("Server: %s", self.smtpserver)
logging.debug("Port: %s", self.serverport)
try:
server = smtplib.SMTP(self.smtpserver, self.serverport,)
if self.starttls:
server.starttls()
server.ehlo()
server.login(self.emailadress, self.password)
#server.sendmail(emailadress, kindle_mail, msg.as_string())
server.sendmail(self.emailadress, kindle_mail, msg)
server.close()
logging.debug("Sent Ebook email to %s ", kindle_mail)
self.send_confirmation(user_mail)
except smtplib.SMTPException as fail:
logging.debug("Could not send email! %s", fail)
# Set Email as Sent
helper.setIsSent(self.mangaid)
logging.info("Sent %s to all requested users.", self.mangatitle)
class TestGitlabClient(unittest.TestCase):
""" Test class for GitlabClient"""
def setUp(self):
self.user = "******"
self.help = Helper()
def test_get_entry(self):
print "Get entry"
userlist = json.loads(json.dumps([
{
"id": 1,
"username": "******",
},
{
"id": 2,
"username": "******",
}
]))
with mock.patch('__builtin__.raw_input', return_value='0'):
assert self.help.get_entry(userlist, 'username') == 1
with mock.patch('__builtin__.raw_input', return_value='1'):
assert self.help.get_entry(userlist, 'username') == 2
with mock.patch('__builtin__.raw_input', return_value='3'):
self.assertRaises(ValueError)
userlist = json.loads(json.dumps([
{
"id": 1,
"username": "******",
}
]))
self.assertEquals( self.help.get_entry(userlist, 'username'), 1)
def test_get_entry_project_id(self):
print "Get entry project id"
projects = json.loads(json.dumps([
{
"id": 4,
"http_url_to_repo": "http://example.com/diaspora/diaspora-client.git",
},
{
"id": 6,
"http_url_to_repo": "http://example.com/brightbox/puppet.git",
},
{
"id": 8,
"http_url_to_repo": "http://example.com/birghtbox/puppet",
}
]))
with mock.patch('__builtin__.raw_input', return_value='0'):
assert self.help.get_entry(projects, 'http_url_to_repo') == 4
with mock.patch('__builtin__.raw_input', return_value='1'):
assert self.help.get_entry(projects, 'http_url_to_repo') == 6
with mock.patch('__builtin__.raw_input', return_value='3'):
self.assertRaises(ValueError)
reponame = "http://example.com/diaspora/diaspora-client.git"
filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo')
self.assertEquals(len(filtered), 1)
self.assertEquals(filtered[0]['http_url_to_repo'], reponame)
projects = json.loads(json.dumps([
{
"id": 4,
"http_url_to_repo": "http://example.com/diaspora/diaspora-client.git",
},
{
"id": 6,
"http_url_to_repo": "http://example.com/diaspora/diaspora",
},
{
"id": 8,
"http_url_to_repo": "http://example.com/diaspora/diaspora-client",
},
{
"id": 10,
"http_url_to_repo": "http://example.com/diaspora/diaspora.git",
}
]))
reponame = "http://example.com/diaspora/diaspora"
filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo')
self.assertEquals(len(filtered), 4)
self.assertEquals(filtered[0]['http_url_to_repo'], "http://example.com/diaspora/diaspora-client.git")
reponame = "http://example.com/diaspora/diaspora.git"
filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo')
self.assertEquals(len(filtered), 1)
self.assertEquals(filtered[0]['http_url_to_repo'], reponame)
reponame = "http://example.com/diaspora/diasporasdfadf.git"
filtered = self.help.filter_list_by_entry(reponame, projects, 'http_url_to_repo')
self.assertEquals(len(filtered), 0)
def test_apicall(self):
print "Test apicall"
text = apicall(test_func, "fsk", "lks", "slkdfj", test="lkjsf")
self.assertEquals(text, "fsk lks slkdfj lkjsf")
def test_get_state_event(self):
print "Test get state event"
with mock.patch('__builtin__.raw_input', return_value='0'):
assert self.help.get_state_event() == "close"
with mock.patch('__builtin__.raw_input', return_value='1'):
assert self.help.get_state_event() == "reopen"
with mock.patch('__builtin__.raw_input', return_value='2'):
assert self.help.get_state_event() == "merge"
with mock.patch('__builtin__.raw_input', return_value='3'):
self.assertRaises(ValueError)
def test_show_info(self):
# not really a test maybe a print test later
print "Test show infos dummy"
entry = {
"id": 4,
"http_url_to_repo": "http://example.com/diaspora/diaspora-client.git",
}
self.help.show_infos(entry, "List", "id", "http_url_to_repo")
def read_arguments(self):
""" function that reads all arguments """
# Get user Input
parser = argparse.ArgumentParser(description='Manga to eManga - m2em')
parser.add_argument(
"-af",
"--add-feed",
help=
"Add RSS Feed of Manga. Only Mangastream & MangaFox are supported")
parser.add_argument("-au",
"--add-user",
help="Adds new user",
action="store_true")
parser.add_argument(
"-lm",
"--list-manga",
help=
"Lists Manga saved in database. If a Manga is passed, lists chapters to said Manga",
nargs="?",
const='all')
parser.add_argument("-lc",
"--list-chapters",
help="Lists the last 10 Chapters",
action="store_true")
parser.add_argument("-Lc",
"--list-chapters-all",
help="Lists all Chapters",
action="store_true")
parser.add_argument("-lf",
"--list-feeds",
help="Lists all feeds",
action="store_true")
parser.add_argument("-lu",
"--list-users",
help="Lists all Users",
action="store_true")
parser.add_argument("-cd",
"--create-db",
help="Creates DB. Uses Configfile for Naming",
action="store_true")
parser.add_argument("-s",
"--start",
help="Starts one loop",
action="store_true")
parser.add_argument(
"--send",
help=
"Sends Chapter directly by chapter ID. Multiple IDs can be given",
default=[],
nargs='*',
)
parser.add_argument(
"--convert",
help=
"Converts Chapter directly by chapter ID. Multiple IDs can be given",
default=[],
nargs='*',
)
parser.add_argument(
"--download",
help=
"Downloads Chapter directly by chapter ID. Multiple IDs can be given",
default=[],
nargs='*',
)
parser.add_argument(
"-p",
"--process",
help=
"Processes chapter(s) by chapter ID, Download, convert, send. Multiple IDs can be given",
default=[],
nargs='*',
)
parser.add_argument(
"-a",
"--action",
help=
"Start action. Options are: rssparser (collecting feed data), downloader, converter or sender "
)
parser.add_argument(
"-ss",
"--switch-send",
help="Pass ID of User. Switches said user Send eBook status")
parser.add_argument("-dc",
"--delete-chapter",
help="Pass ID of Chapter. Deletes said Chapter")
parser.add_argument("-du",
"--delete-user",
help="Pass ID of User. Deletes said User")
parser.add_argument("-df",
"--delete-feed",
help="Pass ID of Feed. Deletes said Feed")
parser.add_argument("--daemon",
help="Run as daemon",
action="store_true")
parser.add_argument("-d",
"--debug",
help="Debug Mode",
action="store_true")
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + __version__)
self.args = parser.parse_args()
# Logging
if self.args.debug:
outputlevel = "debug"
else:
outputlevel = "info"
helper.initialize_logger("log/", outputlevel)
# Check if Arguments are passed or not. At least one is required
if self.args.action is None \
and self.args.add_feed is None \
and self.args.delete_chapter is None \
and self.args.delete_feed is None \
and self.args.delete_user is None \
and self.args.switch_send is None \
and self.args.add_user is False \
and self.args.list_manga is None \
and not any([self.args.add_user,
self.args.create_db,
self.args.daemon,
self.args.list_chapters,
self.args.list_chapters_all,
self.args.list_feeds,
self.args.list_users,
self.args.download,
self.args.convert,
self.args.send,
self.args.process,
self.args.start,]):
logging.error("At least one argument is required!")
logging.debug("Passed arguments: \n %s", self.args)
def list_feeds(self):
helper.printFeeds()
def filter_list(self):
helper.printFilters()
