apkdlcrawler.py

#!/usr/bin/env python3

#
# Required Modules
# - beautifulsoup4
# - html5lib
# - requests
#

import sys
import os
import re
import logging
import multiprocessing
import requests

from bs4 import BeautifulSoup
import unicodedata

from debug import Debug
from apkhelper import ApkVersionInfo
from reporthelper import ReportHelper

###################
# DEBUG VARS      #
###################

# Debug.DEBUG        = True
# Debug.READFROMFILE = True  # Read from file for debugging
# Debug.SAVELASTFILE = True  # Write to file upon each request

###################
# END: DEBUG VARS #
###################

###################
# Globals         #
###################

# logging
logFile   = '{0}.log'.format(os.path.basename(__file__))
logLevel  = (logging.DEBUG if Debug.DEBUG else logging.INFO)
logFormat = '%(asctime)s %(levelname)s/%(funcName)s(%(process)-5d): %(message)s'


class ApkdlCrawler(object):
    def __init__(self, report, dlFiles=[], dlFilesBeta=[]):
        self.report      = report
        self.dlFiles     = dlFiles
        self.dlFilesBeta = dlFilesBeta

    def getUrlFromRedirect(self, apkname, url):
        """
        getUrlFromRedirect(url):
        """
        link      = ''

        session = requests.Session()
        logging.debug('Requesting2: ' + url)
        resp    = session.get(url)
        html    = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')

        try:
            dom  = BeautifulSoup(html, 'html5lib')
            link = dom.find('span', {'class': 'glyphicon glyphicon-cloud-download'}).parent['href']
        except:
            logging.exception('!!! Error parsing html from: "{0}"'.format(url))

        return link
    # END: def getUrlFromRedirect:

    def downloadApk(self, apkInfo, isBeta=False):
        """
        downloadApk(apkInfo): Download the specified URL to APK file name
        """
        apkname = ('beta.' if isBeta else '') + avi.getFilename()

        url     = self.getUrlFromRedirect(apkname, apkInfo.download_src)
        if url == '':
            logging.error('Unable to determine redirect url for ' + apkname)
            return

        logging.info('Downloading "{0}" from: {1}'.format(apkname, url))

        try:
            if os.path.exists(apkname):
                logging.info('{0} already exists'.format(apkname))
                return

            if os.path.exists(os.path.join('.', 'apkcrawler', apkname)):
                logging.info('{0} already exists (in ./apkcrawler/)'.format(apkname))
                return

            if os.path.exists(os.path.join('..', 'apkcrawler', apkname)):
                logging.info('{0} already exists (in ../apkcrawler/)'.format(apkname))
                return

            # Open the url
            session = requests.Session()
            r = session.get(url)

            with open(apkname, 'wb') as local_file:
                local_file.write(r.content)

            logging.debug(('beta:' if isBeta else 'reg :') + apkname)
            return       (('beta:' if isBeta else ''     ) + apkname)
        except OSError:
            logging.exception('!!! Filename is not valid: "{0}"'.format(apkname))
    # END: def downloadApk

    def checkOneApp(self, apkid):
        """
        checkOneApp(apkid):
        """
        logging.info('Checking app: {0}'.format(apkid))

        filenames = []

        url       = 'http://apk-dl.com/' + apkid

        session = requests.Session()
        logging.debug('Requesting: ' + url)
        resp    = session.get(url)
        html    = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')

        try:
            dom     = BeautifulSoup(html, 'html5lib')
            apklist = dom.findAll('ul', {'class': 'apks dlist'})[0]
            apks    = apklist.findAll('div', {'class': 'details'})

            for apk in apks:
                items = apk.findAll('div')
                dApk = {}
                for item in items:
                    itext = '{0}'.format(item.get_text().encode('ascii', 'ignore'))
                    itext = re.sub('\s', '', itext)
                    itextsp = itext.split(':', 1)
                    if len(itextsp) == 2:
                        dApk[str(itextsp[0])] = str(itextsp[1])
                apkurl = apk.find('a', {'class': 'btn btn-success'})
                if apkurl:
                    dApk['url'] = 'http:' + apkurl['href']

                    Debug.printDictionary(dApk)

                    if 'Version' in dApk and 'RequiresAndroid' in dApk:
                        (trash, sdk) = dApk['RequiresAndroid'].split('API:', 1)
                        sdk = sdk[0:-1]
                        (ver, vercode) = dApk['Version'].split('(Code:', 1)
                        ver     = ver.split('(', 1)[0].strip()
                        vercode = vercode[0:-1].strip()

                        avi = ApkVersionInfo(name=apkid,
                                             sdk=sdk,
                                             ver=ver,
                                             vercode=vercode,
                                             download_src=dApk['url'],
                                             crawler_name=self.__class__.__name__
                                             )

                        if self.report.isThisApkNeeded(avi):
                            filenames.append(self.downloadApk(avi))

        except IndexError:
            logging.info('{0} not supported by apk-dl.com ...'.format(apkid))
        except:
            logging.exception('!!! Error parsing html from: "{0}"'.format(url))

        return filenames
    # END: def checkOneApp:

    def crawl(self, threads=5):
        """
        crawl(): check all apk-dl apps
        """
        # Start checking all apkids ...
        p = multiprocessing.Pool(processes=threads, maxtasksperchild=5)  # Run only 5 tasks before re-placing the process
        r = p.map_async(unwrap_self_checkOneApp, list(zip([self] * len(list(self.report.getAllApkIds())), list(self.report.getAllApkIds()))), callback=unwrap_callback)
        r.wait()
        (self.dlFiles, self.dlFilesBeta) = unwrap_getresults()
    # END: crawl():
# END: class ApkdlCrawler

nonbeta = []
beta    = []


def unwrap_callback(results):
    for resultlist in results:
        for result in resultlist:
            if result:
                if result.startswith('beta:'):
                    beta.append(result[5:])
                else:
                    nonbeta.append(result)


def unwrap_getresults():
    return (nonbeta, beta)


def unwrap_self_checkOneApp(arg, **kwarg):
    return ApkdlCrawler.checkOneApp(*arg, **kwarg)


if __name__ == "__main__":
    """
    main(): single parameter for report_sources.sh output
    """
    logging.basicConfig(filename=logFile, filemode='w', level=logLevel, format=logFormat)
    logging.getLogger("requests").setLevel(logging.WARNING)

    lines = ''
    if len(sys.argv[1:]) == 1:
        with open(sys.argv[1]) as report:
            lines = report.readlines()
    else:
        lines = sys.stdin.readlines()

    report = ReportHelper(lines)

    if len(list(report.getAllApkIds())) == 0:
        print('ERROR: expecting:')
        print(' - 1 parameter (report file from output of report_sources.sh)')
        print(' or ')
        print(' - stdin from report_sources.sh')
        exit(1)

    crawler = ApkdlCrawler(report)
    crawler.crawl()

    outputString = ' '.join(crawler.dlFiles)
    if crawler.dlFilesBeta:
        outputString += ' beta ' + ' '.join(crawler.dlFilesBeta)

    if outputString:
        print(outputString)
        sys.stdout.flush()

    logging.debug('Done ...')