def initLogger(self):
        logging.basicConfig(
            level=logging.DEBUG,
            format=
            '%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')

        self.logCounterHandler = LogCounterHandler(logging.WARN)
        self.logger.addHandler(self.logCounterHandler)

        filename = os.path.join(
            self.logDir, '%s.txt' % datetime.datetime.fromtimestamp(
                time.time()).strftime('%Y%m%d_%H%M%S'))
        formatter = logging.Formatter(
            '%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s'
        )
        self.logFileHandler = logging.FileHandler(filename,
                                                  mode='a',
                                                  encoding="utf-8",
                                                  delay=True)
        self.logFileHandler.setLevel(logging.WARN)
        self.logFileHandler.setFormatter(formatter)
        self.logger.addHandler(self.logFileHandler)
    def initLogger(self):
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s',
                            datefmt='%Y-%m-%d %H:%M:%S')

        self.logCounterHandler = LogCounterHandler(logging.WARN)
        self.logger.addHandler(self.logCounterHandler)

        filename = os.path.join(self.logDir,
                                '%s.txt' % datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%H%M%S'))
        formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s')
        self.logFileHandler = logging.FileHandler(filename, mode='a', encoding="utf-8", delay=True)
        self.logFileHandler.setLevel(logging.WARN)
        self.logFileHandler.setFormatter(formatter)
        self.logger.addHandler(self.logFileHandler)
class CrawlerBase(object):
    def __init__(self):
        self.logger = logging.getLogger(LogHelper.LoggerName)
        self.interval = None
        self.downloader = urldownloader.urldownloader()
        self.htmlDir = None
        self.profileDir = None
        self.outputDir = os.getcwd()
        self.debugMode = False
        self.logFileHandler = None
        self.assetDir = None
        self.dictAssetFiles = {}
        self.lastErrorReportTime = time.time()

        #sys.setdefaultencoding("utf-8")
        self.processArgs()
        if not os.path.exists(self.outputDir):
            os.makedirs(self.outputDir)
        self.htmlDir = os.path.join(self.outputDir, "html.files" + os.sep)
        self.assetDir = os.path.join(self.htmlDir, "assets" + os.sep)
        self.profileDir = os.path.join(self.outputDir, "profiles" + os.sep)
        self.logDir = os.path.join(self.outputDir, "logs" + os.sep)
        if not os.path.exists(self.htmlDir):
            os.makedirs(self.htmlDir)
        if not os.path.exists(self.profileDir):
            os.makedirs(self.profileDir)
        if not os.path.exists(self.assetDir):
            os.makedirs(self.assetDir)
        if not os.path.exists(self.logDir):
            os.makedirs(self.logDir)

        self.initLogger()
        self.printArgs()

        self.overallEntriesFile = None
        self.overallEntriesWriter = None
        self.overallEntries = {}
        self.loadOverallEntries()

        self.overallEntriesPendingFile = None
        self.overallEntriesPendingWriter = None
        self.overallEntriesPending = {}
        # self.loadOverallEntriesPending()

    def freeResource(self):
        if (self.logFileHandler):
            self.logFileHandler.close()
        if (self.overallEntriesFile != None):
            self.overallEntriesFile.close()
        if (self.overallEntriesPendingFile != None):
            self.overallEntriesPendingFile.close()

    def initLogger(self):
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s',
                            datefmt='%Y-%m-%d %H:%M:%S')

        self.logCounterHandler = LogCounterHandler(logging.WARN)
        self.logger.addHandler(self.logCounterHandler)

        filename = os.path.join(self.logDir,
                                '%s.txt' % datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%H%M%S'))
        formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s')
        self.logFileHandler = logging.FileHandler(filename, mode='a', encoding="utf-8", delay=True)
        self.logFileHandler.setLevel(logging.WARN)
        self.logFileHandler.setFormatter(formatter)
        self.logger.addHandler(self.logFileHandler)

    def get_attrib(self, node, name, default=None):
        if (node == None):
            return default
        return node.attrib[name] if name in node.attrib else default

    def set_attrib(self, node, name, value):
        if (node == None):
            return
        node.attrib[name] = value

    def find_element_by_xpath(self, element, xpath):
        try:
            nodes = element.xpath(xpath)
            return nodes[0] if len(nodes) > 0 else None
        except:
            return None

    def find_elements_by_xpath(self, element, xpath):
        try:
            nodes = element.xpath(xpath)
            return nodes
        except:
            return None

    def printUsage(self):
        print('%s -h -o <outputDir> --debug --interval=300' % (os.path.basename(sys.argv[0])))

    def printArgs(self):
        self.logger.info("debug=%s" % (self.debugMode))
        self.logger.info("outputDir=%s" % (self.outputDir))
        self.logger.info("htmlDir=%s" % (self.htmlDir))
        self.logger.info("profileDir=%s" % (self.profileDir))
        self.logger.info("interval=%s" % (str(self.interval) if self.interval else "n/a"))
        self.logger.info("")

    def processArgs(self):
        try:
            opts, args = getopt.getopt(sys.argv[1:], "ho:", ["debug", "logfile", "interval="])
            for opt, arg in opts:
                if opt == '-h':
                    self.printUsage()
                    sys.exit()
                elif opt in ("--debug"):
                    self.debugMode = True
                elif opt in ("--interval"):
                    self.interval = int(arg)
                #elif opt in ("-o", "--ofile"):
                elif opt in ("-o"):
                    self.outputDir = arg
        except getopt.GetoptError:
            self.printUsage()
            sys.exit(2)

    def run(self):
        try:
            while True:
                self.parsePages()
                self.sort_csv(os.path.join(self.profileDir, overallFileName),
                              os.path.join(self.profileDir, overallSortedFileName), (0,))
                self.sort_csv(os.path.join(self.profileDir, u"未結案" + os.sep + overallFileName),
                              os.path.join(self.profileDir, u"未結案" + os.sep + overallSortedFileName), (0,))

                if (self.logCounterHandler.logCount > 0 and time.time() - self.lastErrorReportTime > 2 * 60 * 60):
                    dir, scriptName = os.path.split(sys.argv[0])
                    try:
                        EmailHelper.send("%s error report" % (scriptName), self.logCounterHandler.report())
                        self.lastErrorReportTime = time.time()
                        self.logCounterHandler.logCount = 0
                    except Exception as ex:
                        self.logger.exception(ex, "cannot sending email")
                if (self.interval == None):
                    break
                print "press Ctrl+C to exit or wait for %d seconds to continue" % self.interval
                t = time.time()
                while True:
                    time.sleep(1)
                    if (time.time() - t > self.interval):
                        break
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit:
            self.logger.info("SystemExit")
        except Exception as ex:
            self.logger.exception(ex, "Unknown error")
        self.freeResource()
        self.logger.info("bye!")

    def parsePages(self):
        pass

    def loadOverallEntries(self):
        filename = os.path.join(self.profileDir, overallFileName)
        if (os.path.isfile(filename)):
            with open(filename, 'rb') as csvfile:
                for row in csv.reader(csvfile):
                    if (len(row) > 0):
                        self.overallEntries[row[0]] = True

    def saveOverallEntry(self, id, values):
        if (id in self.overallEntries):
            return
        self.overallEntries[id] = True
        #self.logger.warn("saveOverallEntry -- overallEntry.title: %s" % overallEntry.title)

        if (self.overallEntriesWriter == None or self.overallEntriesFile == None):
            self.overallEntriesFile = open(os.path.join(self.profileDir, overallFileName), "ab")
            self.overallEntriesWriter = csv.writer(self.overallEntriesFile)
        self.overallEntriesWriter.writerow(values)
        self.overallEntriesFile.flush()

    def loadOverallEntriesPending(self):
        filename = os.path.join(self.profileDir, u"未結案" + os.sep + overallFileName)
        if (os.path.isfile(filename)):
            with open(filename, 'rb') as csvfile:
                for row in csv.reader(csvfile):
                    if (len(row) > 0):
                        self.overallEntriesPending[row[0]] = True

    def saveOverallEntryPending(self, id, values):
        if (id in self.overallEntriesPending):
            return
        self.overallEntriesPending[id] = True
        #self.logger.warn("saveOverallEntryPending -- overallEntryPending.title: %s" % overallEntryPending.title)

        if (self.overallEntriesPendingWriter == None or self.overallEntriesPendingFile == None):
            self.overallEntriesPendingFile = open(os.path.join(self.profileDir, u"未結案" + os.sep + overallFileName), "ab")
            self.overallEntriesPendingWriter = csv.writer(self.overallEntriesPendingFile)
        self.overallEntriesPendingWriter.writerow(values)
        self.overallEntriesPendingFile.flush()

    def sort_csv(self, csvFilename, dstFilename, sort_key_columns):
        data = []
        with open(csvFilename, 'rb') as f:
            for row in csv.reader(f):
                data.append(row)
                #data.append(convert(types, row))
        data.sort(key=operator.itemgetter(*sort_key_columns))
        with open(dstFilename, 'wb') as f:
            csv.writer(f).writerows(data)
class CrawlerBase(object):
    def __init__(self):
        self.logger = logging.getLogger(LogHelper.LoggerName)
        self.interval = None
        self.downloader = urldownloader.urldownloader()
        self.htmlDir = None
        self.profileDir = None
        self.outputDir = os.getcwd()
        self.debugMode = False
        self.logFileHandler = None
        self.assetDir = None
        self.dictAssetFiles = {}
        self.lastErrorReportTime = time.time()

        #sys.setdefaultencoding("utf-8")
        self.processArgs()
        if not os.path.exists(self.outputDir):
            os.makedirs(self.outputDir)
        self.htmlDir = os.path.join(self.outputDir, "html.files" + os.sep)
        self.assetDir = os.path.join(self.htmlDir, "assets" + os.sep)
        self.profileDir = os.path.join(self.outputDir, "profiles" + os.sep)
        self.logDir = os.path.join(self.outputDir, "logs" + os.sep)
        if not os.path.exists(self.htmlDir):
            os.makedirs(self.htmlDir)
        if not os.path.exists(self.profileDir):
            os.makedirs(self.profileDir)
        if not os.path.exists(self.assetDir):
            os.makedirs(self.assetDir)
        if not os.path.exists(self.logDir):
            os.makedirs(self.logDir)

        self.initLogger()
        self.printArgs()

        self.overallEntriesFile = None
        self.overallEntriesWriter = None
        self.overallEntries = {}
        self.loadOverallEntries()

        self.overallEntriesPendingFile = None
        self.overallEntriesPendingWriter = None
        self.overallEntriesPending = {}
        # self.loadOverallEntriesPending()

    def freeResource(self):
        if (self.logFileHandler):
            self.logFileHandler.close()
        if (self.overallEntriesFile != None):
            self.overallEntriesFile.close()
        if (self.overallEntriesPendingFile != None):
            self.overallEntriesPendingFile.close()

    def initLogger(self):
        logging.basicConfig(
            level=logging.DEBUG,
            format=
            '%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')

        self.logCounterHandler = LogCounterHandler(logging.WARN)
        self.logger.addHandler(self.logCounterHandler)

        filename = os.path.join(
            self.logDir, '%s.txt' % datetime.datetime.fromtimestamp(
                time.time()).strftime('%Y%m%d_%H%M%S'))
        formatter = logging.Formatter(
            '%(asctime)s %(levelname)s %(filename)s[line:%(lineno)d] %(message)s'
        )
        self.logFileHandler = logging.FileHandler(filename,
                                                  mode='a',
                                                  encoding="utf-8",
                                                  delay=True)
        self.logFileHandler.setLevel(logging.WARN)
        self.logFileHandler.setFormatter(formatter)
        self.logger.addHandler(self.logFileHandler)

    def get_attrib(self, node, name, default=None):
        if (node == None):
            return default
        return node.attrib[name] if name in node.attrib else default

    def set_attrib(self, node, name, value):
        if (node == None):
            return
        node.attrib[name] = value

    def find_element_by_xpath(self, element, xpath):
        try:
            nodes = element.xpath(xpath)
            return nodes[0] if len(nodes) > 0 else None
        except:
            return None

    def find_elements_by_xpath(self, element, xpath):
        try:
            nodes = element.xpath(xpath)
            return nodes
        except:
            return None

    def printUsage(self):
        print('%s -h -o <outputDir> --debug --interval=300' %
              (os.path.basename(sys.argv[0])))

    def printArgs(self):
        self.logger.info("debug=%s" % (self.debugMode))
        self.logger.info("outputDir=%s" % (self.outputDir))
        self.logger.info("htmlDir=%s" % (self.htmlDir))
        self.logger.info("profileDir=%s" % (self.profileDir))
        self.logger.info("interval=%s" %
                         (str(self.interval) if self.interval else "n/a"))
        self.logger.info("")

    def processArgs(self):
        try:
            opts, args = getopt.getopt(sys.argv[1:], "ho:",
                                       ["debug", "logfile", "interval="])
            for opt, arg in opts:
                if opt == '-h':
                    self.printUsage()
                    sys.exit()
                elif opt in ("--debug"):
                    self.debugMode = True
                elif opt in ("--interval"):
                    self.interval = int(arg)
                #elif opt in ("-o", "--ofile"):
                elif opt in ("-o"):
                    self.outputDir = arg
        except getopt.GetoptError:
            self.printUsage()
            sys.exit(2)

    def run(self):
        try:
            while True:
                self.parsePages()
                self.sort_csv(
                    os.path.join(self.profileDir, overallFileName),
                    os.path.join(self.profileDir, overallSortedFileName),
                    (0, ))
                self.sort_csv(
                    os.path.join(self.profileDir,
                                 u"未結案" + os.sep + overallFileName),
                    os.path.join(self.profileDir,
                                 u"未結案" + os.sep + overallSortedFileName),
                    (0, ))

                if (self.logCounterHandler.logCount > 0 and
                        time.time() - self.lastErrorReportTime > 2 * 60 * 60):
                    dir, scriptName = os.path.split(sys.argv[0])
                    try:
                        EmailHelper.send("%s error report" % (scriptName),
                                         self.logCounterHandler.report())
                        self.lastErrorReportTime = time.time()
                        self.logCounterHandler.logCount = 0
                    except Exception as ex:
                        self.logger.exception(ex, "cannot sending email")
                if (self.interval == None):
                    break
                print "press Ctrl+C to exit or wait for %d seconds to continue" % self.interval
                t = time.time()
                while True:
                    time.sleep(1)
                    if (time.time() - t > self.interval):
                        break
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit:
            self.logger.info("SystemExit")
        except Exception as ex:
            self.logger.exception(ex, "Unknown error")
        self.freeResource()
        self.logger.info("bye!")

    def parsePages(self):
        pass

    def loadOverallEntries(self):
        filename = os.path.join(self.profileDir, overallFileName)
        if (os.path.isfile(filename)):
            with open(filename, 'rb') as csvfile:
                for row in csv.reader(csvfile):
                    if (len(row) > 0):
                        self.overallEntries[row[0]] = True

    def saveOverallEntry(self, id, values):
        if (id in self.overallEntries):
            return
        self.overallEntries[id] = True
        #self.logger.warn("saveOverallEntry -- overallEntry.title: %s" % overallEntry.title)

        if (self.overallEntriesWriter == None
                or self.overallEntriesFile == None):
            self.overallEntriesFile = open(
                os.path.join(self.profileDir, overallFileName), "ab")
            self.overallEntriesWriter = csv.writer(self.overallEntriesFile)
        self.overallEntriesWriter.writerow(values)
        self.overallEntriesFile.flush()

    def loadOverallEntriesPending(self):
        filename = os.path.join(self.profileDir,
                                u"未結案" + os.sep + overallFileName)
        if (os.path.isfile(filename)):
            with open(filename, 'rb') as csvfile:
                for row in csv.reader(csvfile):
                    if (len(row) > 0):
                        self.overallEntriesPending[row[0]] = True

    def saveOverallEntryPending(self, id, values):
        if (id in self.overallEntriesPending):
            return
        self.overallEntriesPending[id] = True
        #self.logger.warn("saveOverallEntryPending -- overallEntryPending.title: %s" % overallEntryPending.title)

        if (self.overallEntriesPendingWriter == None
                or self.overallEntriesPendingFile == None):
            self.overallEntriesPendingFile = open(
                os.path.join(self.profileDir,
                             u"未結案" + os.sep + overallFileName), "ab")
            self.overallEntriesPendingWriter = csv.writer(
                self.overallEntriesPendingFile)
        self.overallEntriesPendingWriter.writerow(values)
        self.overallEntriesPendingFile.flush()

    def sort_csv(self, csvFilename, dstFilename, sort_key_columns):
        data = []
        with open(csvFilename, 'rb') as f:
            for row in csv.reader(f):
                data.append(row)
                #data.append(convert(types, row))
        data.sort(key=operator.itemgetter(*sort_key_columns))
        with open(dstFilename, 'wb') as f:
            csv.writer(f).writerows(data)