Exemplo n.º 1
0
Arquivo: views.py Projeto: sjf/ana
def is_valid_submission(json_data):
    try:
        #jsonschema.validate(json_data, config.SUBMIT_SCHEMA)
        return True
    except jsonschema.ValidationError as e:
        log("Invalid client response:", e, json_data)
        return False
Exemplo n.º 2
0
Arquivo: game.py Projeto: sjf/ana
    def get_game_for_client(self, client_id):
        # Check if client is already in active game
        game_state = self.db.get_client_game(client_id)
        if game_state:
            #log('Client in game', game_state.id)
            if game_state and game_state.end_time > lib.now():
                if game_state.has_client(client_id):
                    client = game_state.client(client_id)
                else:
                    client = ClientState(client_id)
                    self.db.add_client_to_game(game_state, client)
                return game_state, client

        # Try to add to the client to the most recently started game
        minimum_end_time = lib.now() + config.MIN_TIME_LEFT_MS
        game_state = self.db.get_latest_game(minimum_end_time)
        if not game_state:
            # Start a new game for the client
            game_state = self._new_game()
        else:
            log('Add client to existing game', game_state.id)

        log('Adding client to game ', game_state.id)
        client = ClientState(client_id)
        game_state = self.db.add_client_to_game(game_state, client)
        return game_state, client
Exemplo n.º 3
0
    def trace_ids_iterator(self,
                           start: datetime,
                           end: datetime,
                           filter_expr: str = 'ok or !ok'):

        if not self.__less_than_24_hours(start, end):
            end = start + timedelta(seconds=23 * 60 * 60)

        response = self.get_trace_summaries(start=start,
                                            end=end,
                                            filter_expression=filter_expr)
        next_token = response.get('NextToken', None)
        while True:
            summaries = response["TraceSummaries"]
            log(f"Fetching {len(summaries)} trace summaries IDs")
            for summary in summaries:
                yield summary['Id']

            if next_token is None:
                break
            summaries = self.get_trace_summaries(start=start,
                                                 end=end,
                                                 filter_expression=filter_expr,
                                                 next_token=next_token)
            next_token = summaries.get('NextToken', None)
Exemplo n.º 4
0
def rows2txt():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize + 94
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                path = mkdir(item["id"])
                chs = db.get_chapters(item["id"])
                if chs is not None:
                    for ch in chs:
                        # db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
                        chtext = db.get_chapter_text(ch["id"])
                        if chtext is not None:
                            try:
                                file_path = os.path.join(path, str(ch["id"]))
                                text = htmlstrip(chtext["text"])
                                # write to file
                                write2file(file_path, text)
                                # update chapter size and text
                                db.update_chapter_size(ch["id"], len(text))
                                db.update_chapter_text(ch["id"], text)
                                print ch["id"]
                            except Exception, ex:
                                log("Error: ch " + str(ch["id"]) + " " + str(Exception) + ":" + str(ex))
Exemplo n.º 5
0
Arquivo: ddw.py Projeto: wangjun/novel
def save_book(id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is None:
        print '-------------begin ' + str(id) + '-------------'
        book = get_book_detail(id)

        if book is not None:
            print '-------------saving-------------'
            #author
            author_id = 0
            author = dw.get_author(book['author'])
            if author is None:
                author_id = dw.insert_author(book['author'])
            else:
                author_id = int(author['id'])
            #category
            cate_id = 0
            cate = dw.get_category(book['cate'])
            if cate is None:
                cate_id = dw.insert_category(book['cate'])
            else:
                cate_id = int(cate['id'])
            #book
            desc = book['desc']
            try:
                desc = htmlstrip(desc).strip()
            except Exception, ex:
                log("Error: desc.error")
            dw.insert_book(id, book['name'], desc, author_id, cate_id,
                           book['is_finish'], book['recommend_count'])

            print '-------------end of ' + str(id) + '-----------'
Exemplo n.º 6
0
Arquivo: ddw.py Projeto: wangjun/novel
def get_chapter(book_id, id):
    try:
        _url = __url__ + "%d/%d/%d.html" % (book_id / 1000, book_id, id)
        print _url

        html = urllib2.urlopen(_url).read()
        """
        soup = BeautifulSoup(html,fromEncoding="gbk")

        info = soup.findAll(attrs={'id' : "booktext"})[0]
        info.strong.extract()
        s = info.prettify()
        s = s.replace('<div id="booktext">\n<!--go-->','')
        s = s.replace('<!--over-->\n</div>','')
        s = s.strip()
        print s
        return unicode(s, 'utf-8', "ignore")
        """
        html = unicode(html, __encode__, "ignore")
        m = re.compile(r'<!--go-->(?P<T>.*?)<!--over-->', re.S).search(html)
        text = ''
        if m:
            text = m.group('T')
            text = re.sub(r"(<STRONG>.*</STRONG><br><br>)", "", text)
        return text.strip()

    except urllib2.HTTPError:
        log("Error: get_chapter book_id " + str(book_id) + " not exists.")
    except Exception, ex:
        log("Error: get_chapter book_id " + str(book_id) + " " +
            str(Exception) + ":" + str(ex))
Exemplo n.º 7
0
def read_file(fobj_read, delimiter=';', encoding='utf-8-sig', skip=0):
    def get_real_header(header):
        """ Get real header cut at the first empty column """
        new_header = []
        for head in header:
            if head:
                new_header.append(head)
            else:
                break
        return new_header

    def check_id_column(header):
        try:
            header.index('id')
        except ValueError as ve:
            log_error("No External Id (id) column defined, please add one")
            raise ve

    def skip_line(reader):
        log_info("Skipping until line %s excluded" % skip)
        for _ in xrange(1, skip):
            reader.next()

    log('open %s' % fobj_read.name)
    reader = UnicodeReader(fobj_read,
                           delimiter=delimiter,
                           encoding='utf-8-sig')
    header = reader.next()
    header = get_real_header(header)
    check_id_column(header)
    skip_line(reader)
    data = [l for l in reader]
    return header, data
Exemplo n.º 8
0
def rows2txt ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize+94
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                path = mkdir(item["id"])
                chs = db.get_chapters(item['id'])
                if chs is not None:
                    for ch in chs:
                        #db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
                        chtext = db.get_chapter_text(ch['id'])
                        if chtext is not None:
                            try:
                                file_path = os.path.join(path, str(ch['id']))
                                text = htmlstrip(chtext["text"])
                                #write to file
                                write2file(file_path, text)
                                #update chapter size and text
                                db.update_chapter_size(ch['id'], len(text))
                                db.update_chapter_text(ch['id'], text)
                                print ch['id']
                            except Exception,ex:
                                log("Error: ch "+ str(ch['id']) + " " + str(Exception) + ":" + str(ex) )
Exemplo n.º 9
0
def do_page(page, temp):
    print page
    params = dictify(temp)
    if not 'sesso' in params:
        return
    sex = params['sesso'].lower()
    if sex == 'm':
        target = male
    elif sex == 'f':
        target = female
    else:
        lib.log('sesso', sex, page)
        return

    item = pywikibot.ItemPage.fromPage(page)
    if not item.exists():
        item = wdapi.createItem(page)
    prop.setTarget(target)
    add, reason = wdapi.canClaimBeAdded(item, prop, checkDupe=True)
    if add:
        print 'Adding a claim...'
        item.addClaim(prop)
        prop.addSource(lib.claim)
    else:
        print 'Not adding because: "{0}"'.format(reason)
Exemplo n.º 10
0
 def iterator(self):
     log('Parsing inputs')
     sqs_event = AwsSqsEvent(event=self.input_data)
     for record in sqs_event.each_record():
         sqs_record = AwsSqsEventRecord(record)
         body = json.loads(sqs_record.get_body())
         yield body
Exemplo n.º 11
0
def get_chapter(book_id, id):
    try:
        _url = __url__ + "%d/%d/%d.html" % (book_id/1000, book_id, id)
        print _url

        html = urllib2.urlopen(_url).read()
        """
        soup = BeautifulSoup(html,fromEncoding="gbk")

        info = soup.findAll(attrs={'id' : "booktext"})[0]
        info.strong.extract()
        s = info.prettify()
        s = s.replace('<div id="booktext">\n<!--go-->','')
        s = s.replace('<!--over-->\n</div>','')
        s = s.strip()
        print s
        return unicode(s, 'utf-8', "ignore")
        """
        html = unicode(html, __encode__, "ignore")
        m = re.compile(r'<!--go-->(?P<T>.*?)<!--over-->', re.S).search(html)
        text = ''
        if m:
            text = m.group('T')
            text = re.sub(r"(<STRONG>.*</STRONG><br><br>)", "", text)
        return text.strip()

    except urllib2.HTTPError:
        log("Error: get_chapter book_id "+ str(book_id) + " not exists.")
    except Exception,ex:
        log("Error: get_chapter book_id "+ str(book_id) + " " + str(Exception) + ":" + str(ex) )
Exemplo n.º 12
0
def save_book (id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is None:
        print '-------------begin '+ str(id)+'-------------'
        book = get_book_detail(id)

        if book is not None:
            print '-------------saving-------------'
            #author
            author_id = 0
            author = dw.get_author(book['author'])
            if author is None:
                author_id = dw.insert_author(book['author'])
            else:
                author_id = int(author['id'])
            #category
            cate_id = 0
            cate = dw.get_category(book['cate'])
            if cate is None:
                cate_id = dw.insert_category(book['cate'])
            else:
                cate_id = int(cate['id'])
            #book
            desc = book['desc']
            try:
                desc = htmlstrip(desc).strip()
            except Exception,ex:
                log("Error: desc.error")
            dw.insert_book(id, book['name'], desc, author_id, cate_id, book['is_finish'],book['recommend_count'])

            print '-------------end of '+ str(id)+'-----------'
Exemplo n.º 13
0
def do_page(page, temp):
    print page
    params = dictify(temp)
    if not 'sesso' in params:
        return
    sex = params['sesso'].lower()
    if sex == 'm':
        target = male
    elif sex == 'f':
        target = female
    else:
        lib.log('sesso', sex, page)
        return

    item = pywikibot.ItemPage.fromPage(page)
    if not item.exists():
        item = wdapi.createItem(page)
    prop.setTarget(target)
    add, reason = wdapi.canClaimBeAdded(item, prop, checkDupe=True)
    if add:
        print 'Adding a claim...'
        item.addClaim(prop)
        prop.addSource(lib.claim)
    else:
        print 'Not adding because: "{0}"'.format(reason)
Exemplo n.º 14
0
    def __init__(self, parent=None):
        QWidget.__init__(self, parent)
        self.ui = MainWindow.Ui_MainWindow()
        self.ui.setupUi(self)

        # Translations
        try:
            if os.path.exists(os.getcwd() + "/translations/"):
                dirr = os.getcwd() + "/translations/"
            else:
                dirr = '/usr/share/urtdsc/'

            translator = QTranslator(app)
            translator.load("urtdsc-" + QLocale.system().name() + ".qm", dirr)
            app.installTranslator(translator)
            lib.log(
                '1', "Loaded translation for: " + QLocale.system().name() +
                " from " + dirr)
        except:
            lib.log('1', "Cant load translations!")

        # Settings
        self.settings = QSettings('urtdsc', 'settings')
        self.restoreGeometry(self.settings.value('geometry').toByteArray())
        self.restoreState(self.settings.value('state').toByteArray())

        if os.path.exists(os.path.expanduser('~/.q3a/q3ut4/demos')):
            pass
        else:
            nodemos = '1'

        self.fillDemosList(1)

        self.ui.label.setText("")

        # We need this for translations >.<
        self.ui.copyDemoToDesktop.setText(self.tr("Copy demo to desktop"))
        self.ui.viewAllScreenshots.setText(self.tr("View all screenshots"))
        self.ui.createArchive.setText(self.tr("Create archive..."))
        self.ui.menu_File.setTitle(self.tr("&File"))
        self.ui.action_Options.setText(self.tr("&Options"))
        self.ui.action_Exit.setText(self.tr("&Exit"))
        self.ui.menu_Demos.setTitle(self.tr("&Demos"))
        self.ui.action_Update_list.setText(self.tr("&Update list"))
        self.ui.menu_Help.setTitle(self.tr("&Help"))
        self.ui.action_About_urtdsc.setText(self.tr("About &UrTDSC..."))
        self.ui.actionAbout_Qt.setText(self.tr("About &Qt..."))

        QMetaObject.connectSlotsByName(self)
        self.ui.demosList.itemClicked.connect(self.showDemoInfo)
        self.ui.demosList.itemSelectionChanged.connect(self.showDemoInfo)
        self.ui.demosList.itemActivated.connect(self.showDemoInfo)
        self.ui.demosList.customContextMenuRequested.connect(
            self.demoContextMenu)
        self.ui.copyDemoToDesktop.clicked.connect(self.copydemos)
        self.ui.viewAllScreenshots.clicked.connect(self.allScreenshotsDialog)
        self.ui.action_Update_list.triggered.connect(self.fillDemosList)
        self.ui.action_About_urtdsc.triggered.connect(self.aboutWindow)
        self.ui.actionAbout_Qt.triggered.connect(self.aboutQt)
        self.ui.action_Exit.triggered.connect(self.close)
Exemplo n.º 15
0
 def execute(self, sql, params=()):
     try:
         cur = self.connection.cursor()
         return cur.execute(sql, params)
     except Exception as e:
         lib.log("Error at SQL")
         lib.log(sql)
         raise e
Exemplo n.º 16
0
def alert(msg, level):
    """
    alert user
    @param msg string the message to send
    @param grade int error level 1: fatal 2: warning
    @return null
    """
    lib.log(msg, '/data0/logs/mysqlmond.pid')
Exemplo n.º 17
0
    def test_post(self):

        splunk.access_token = ''
        splunk.project_id = ''

        splunk.log({
          'event': 'event',
          'duration': '5ms'
        })
Exemplo n.º 18
0
 def restartSeq(self, seqname):
     try:
         cur = self.connection.cursor()
         sql = "alter sequence %s restart with 1" % seqname
         return cur.execute(sql)
     except Exception as e:
         lib.log("Error at SQL")
         lib.log(sql)
         raise e
Exemplo n.º 19
0
 def truncateTable(self, tablename):
     try:
         cur = self.connection.cursor()
         sql = "truncate table %s;" % tablename
         return cur.execute(sql)
     except Exception as e:
         lib.log("Error at SQL")
         lib.log(sql)
         raise e
Exemplo n.º 20
0
    def __init__(self, parent=None):
        QWidget.__init__(self, parent)
        self.ui = MainWindow.Ui_MainWindow()
        self.ui.setupUi(self)

        # Translations
        try:
            if os.path.exists(os.getcwd() + "/translations/"):
                dirr = os.getcwd() + "/translations/"
            else:
                dirr = '/usr/share/urtdsc/'
            
            translator = QTranslator(app)
            translator.load("urtdsc-" + QLocale.system().name() + ".qm", dirr)
            app.installTranslator(translator)
            lib.log('1', "Loaded translation for: " + QLocale.system().name() + " from " + dirr)
        except:
            lib.log('1', "Cant load translations!")

        # Settings
        self.settings = QSettings('urtdsc', 'settings')
        self.restoreGeometry(self.settings.value('geometry').toByteArray())
        self.restoreState(self.settings.value('state').toByteArray())

        if os.path.exists(os.path.expanduser('~/.q3a/q3ut4/demos')):
            pass
        else:
            nodemos = '1'

        self.fillDemosList(1)

        self.ui.label.setText("")

        # We need this for translations >.<
        self.ui.copyDemoToDesktop.setText(self.tr("Copy demo to desktop"))
        self.ui.viewAllScreenshots.setText(self.tr("View all screenshots"))
        self.ui.createArchive.setText(self.tr("Create archive..."))
        self.ui.menu_File.setTitle(self.tr("&File"))
        self.ui.action_Options.setText(self.tr("&Options"))
        self.ui.action_Exit.setText(self.tr("&Exit"))
        self.ui.menu_Demos.setTitle(self.tr("&Demos"))
        self.ui.action_Update_list.setText(self.tr("&Update list"))
        self.ui.menu_Help.setTitle(self.tr("&Help"))
        self.ui.action_About_urtdsc.setText(self.tr("About &UrTDSC..."))
        self.ui.actionAbout_Qt.setText(self.tr("About &Qt..."))

        QMetaObject.connectSlotsByName(self)
        self.ui.demosList.itemClicked.connect(self.showDemoInfo)
        self.ui.demosList.itemSelectionChanged.connect(self.showDemoInfo)
        self.ui.demosList.itemActivated.connect(self.showDemoInfo)
        self.ui.demosList.customContextMenuRequested.connect(self.demoContextMenu)
        self.ui.copyDemoToDesktop.clicked.connect(self.copydemos)
        self.ui.viewAllScreenshots.clicked.connect(self.allScreenshotsDialog)
        self.ui.action_Update_list.triggered.connect(self.fillDemosList)
        self.ui.action_About_urtdsc.triggered.connect(self.aboutWindow)
        self.ui.actionAbout_Qt.triggered.connect(self.aboutQt)
        self.ui.action_Exit.triggered.connect(self.close)
Exemplo n.º 21
0
def collect(hosts):
  if ping_method == 'fping':
    responses = fping(hosts)
  else:
    responses = ping(hosts)

  for (host, delay) in responses:
    add_value(host, delay)
    log('collecting host ' + host + ': delay ' + delay, 1)
Exemplo n.º 22
0
def parse_ping_response(string):
  if string.find('avg') == -1:
    log('ping: no answer, ping output was' + string, 2)
    return 'U'

  stats_start = string.find('=')
  avg_start = string.find('/', stats_start)+1
  avg_end = string.find('/', avg_start)
  return string[avg_start:avg_end]
Exemplo n.º 23
0
def text ():
    _count = 22962
    for i in range(2900, _count+1):
        #if i%10==1:
            #print "sleeping..."
            #time.sleep(2)
        try:
            save_text(i)
        except Exception,ex:
            log("Error: chapters book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
Exemplo n.º 24
0
def ClientRead(data):
    opcode_and_fin = data[0]
    msg_len = data[1] - 128
    mask = data[2:6]
    encrypted_msg = data[6: 6+msg_len]
    msg = bytearray([encrypted_msg[i] ^ mask[i % 4]
                     for i in range(msg_len)])
    clientMsg = str(msg, "utf-8")
    log('Incoming message: ' + str(clientMsg))
    return clientMsg
Exemplo n.º 25
0
def books ():
    _count = 24221
    for i in range(24154, _count+1):
        if i%50==1:
            print "sleeping..."
            time.sleep(2)
        try:
            save_book(i)
        except Exception,ex:
            log("Error: book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
Exemplo n.º 26
0
def chapters ():
    _count = 24221
    for i in range(24154, _count+1):
        #if i%10==1:
            #print "sleeping..."
            #time.sleep(2)
        try:
            save_chapters(i)
        except Exception,ex:
            log("Error: chapters book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
Exemplo n.º 27
0
def split_sort(split, header, data):
    split_index = 0
    if split:
        try:
            split_index = header.index(split)
        except ValueError as ve:
            log("column %s not defined" % split)
            raise ve
        data = sorted(data, key=lambda d: d[split_index])
    return data, split_index
Exemplo n.º 28
0
Arquivo: ddw.py Projeto: wangjun/novel
def chapters():
    _count = 24221
    for i in range(24154, _count + 1):
        #if i%10==1:
        #print "sleeping..."
        #time.sleep(2)
        try:
            save_chapters(i)
        except Exception, ex:
            log("Error: chapters book_id " + str(i) + " " + str(Exception) +
                ":" + str(ex))
Exemplo n.º 29
0
 def captureAndUpload(self, imgName):
     localtime = str(time.strftime("%H%M%S", time.localtime()))
     homeImgName = imgName + '_' + localtime + '.png'
     self.driver.save_screenshot(homeImgName)
     localImg = os.path.abspath(homeImgName)
     self.localImgList.append(localImg)
     ossPath = self.getOssPath(homeImgName)
     fullOssPath = self.cdnHost + '/' + ossPath
     self.ossImgList.append(fullOssPath)
     upload(ossPath, localImg)
     log('异常截图已上传: ' + fullOssPath)
Exemplo n.º 30
0
Arquivo: ddw.py Projeto: wangjun/novel
def text():
    _count = 22962
    for i in range(2900, _count + 1):
        #if i%10==1:
        #print "sleeping..."
        #time.sleep(2)
        try:
            save_text(i)
        except Exception, ex:
            log("Error: chapters book_id " + str(i) + " " + str(Exception) +
                ":" + str(ex))
Exemplo n.º 31
0
Arquivo: ddw.py Projeto: wangjun/novel
def books():
    _count = 24221
    for i in range(24154, _count + 1):
        if i % 50 == 1:
            print "sleeping..."
            time.sleep(2)
        try:
            save_book(i)
        except Exception, ex:
            log("Error: book_id " + str(i) + " " + str(Exception) + ":" +
                str(ex))
Exemplo n.º 32
0
def parse_fping_response(string):
  end_host = string.find(' ')
  start_delay = string.find(':')+2
  host = string[0:end_host]
  
  if string.find('avg') == -1:
    log('fping: no answer, fping output was' + string, 2)
    return (host, 'U')
  
  stats_start = string.find('max')+5
  avg_start = string.find('/', stats_start)+1
  avg_end = string.find('/', avg_start)
  return (host, string[avg_start:avg_end])
Exemplo n.º 33
0
    def on_event(self, event, extension):
        try:
            vrs = extension.variables
            data = event.get_data()

            if "reset" in data:
                log("LIST DOCSETS RESET", LogType.DEBUG)
                # Unselect docset
                extension.unselect_docset()
                # List docsets
                return RenderResultListAction(
                    extension.get_sorted_docsets(vrs.query))

            log("QUERY IN DOCSET", LogType.DEBUG)
            # Select a docset
            extension.select_docset(data)
            # Reset query to allow keyword search in the docset
            vrs.query = ""
            # Update UI query string
            extension.update_ui_query_string(event, vrs.query)
            # you may want to return another list of results
            return RenderResultListAction([
                ResultItems.docset_searchforin_result(vrs.sel_docset,
                                                      vrs.query),
                ResultItems.docset_change_result(),
            ])
        except Exception as error:  # pylint: disable=broad-except
            log(error, LogType.ERROR)
            log(traceback.format_exc(), LogType.DEBUG)
Exemplo n.º 34
0
Arquivo: ddw.py Projeto: wangjun/novel
def get_chapters(id):
    try:
        _url = __url__ + "%d/%d/index.html" % ((id / 1000), id)
        print _url
        html = urllib2.urlopen(_url).read()

        soup = BeautifulSoup(html, fromEncoding="gbk")

        a = dict()

        volumes = soup.findAll(attrs={'class': re.compile("^book_article_")})
        flag = False
        i = 0
        j = 0
        sort = 0
        count = len(volumes)
        _re = re.compile(r'(?P<T>\d+?)\.html')
        for v in volumes:

            if v.attrs[0][1] == "book_article_texttitle":
                a[i] = {}
                a[i]['title'] = v.contents[0]
                a[i]['chs'] = []
                flag = True

            if v.attrs[0][1] == "book_article_listtext" and flag == True:
                chs = v.findAll({"a": True})
                for x in chs:
                    id = int(_re.search(x.attrs[0][1]).group("T"))
                    title = x.contents[0]
                    var = id, title, sort
                    a[i]['chs'].append(var)
                    sort += 1

            if j < count - 1:
                if volumes[j + 1].attrs[0][1] == "book_article_texttitle":
                    flag = False
                    i += 1

            j += 1

        return a
    except urllib2.HTTPError:
        log("Error: get_chapters book_id " + str(id) + " not exists.")
    except Exception, ex:
        log("Error: get_chapters book_id " + str(id) + " " + str(Exception) +
            ":" + str(ex))
Exemplo n.º 35
0
def get_chapters (id):
    try:
        _url = __url__ + "%d/%d/index.html" % ((id/1000), id)
        print _url
        html = urllib2.urlopen(_url).read()

        soup = BeautifulSoup(html,fromEncoding="gbk")

        a = dict()

        volumes = soup.findAll(attrs={'class' : re.compile("^book_article_")})
        flag = False
        i = 0
        j = 0
        sort = 0
        count = len(volumes)
        _re = re.compile(r'(?P<T>\d+?)\.html')
        for v in volumes:

            if v.attrs[0][1] == "book_article_texttitle":
                a[i] = {}
                a[i]['title'] = v.contents[0]
                a[i]['chs'] = []
                flag = True


            if v.attrs[0][1] == "book_article_listtext" and flag == True:
                chs = v.findAll({"a":True})
                for x in chs:
                    id = int(_re.search(x.attrs[0][1]).group("T"))
                    title = x.contents[0]
                    var = id, title, sort
                    a[i]['chs'].append(var)
                    sort += 1

            if j < count-1 :
                if volumes[j+1].attrs[0][1] == "book_article_texttitle":
                    flag = False
                    i += 1

            j += 1

        return a
    except urllib2.HTTPError:
        log("Error: get_chapters book_id "+ str(id) + " not exists.")
    except Exception,ex:
        log("Error: get_chapters book_id "+ str(id) + " " + str(Exception) + ":" + str(ex) )
Exemplo n.º 36
0
Arquivo: ddw.py Projeto: wangjun/novel
def get_book_detail(id):
    try:
        _url = __url__ + "info/%d/%d.htm" % ((id / 1000), id)
        html = urllib2.urlopen(_url).read()
        #html = unicode(html, __encode__, "ignore")

        soup = BeautifulSoup(html)
        info = soup.findAll(id=['In'])[0]

        #print type(info)
        #print info.__dict__
        #print info.prettify()

        a = dict()

        m = re.compile(r'作 者:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['author'] = m.group('T').strip()

        m = re.compile(r'类 别:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['cate'] = m.group('T').strip()

        m = re.compile(r'状 态:(?P<T>.*?)<', re.S).search(info.prettify())
        if m:
            tmp = m.group('T').strip()
            a['is_finish'] = False
            if tmp == "已完成":
                a['is_finish'] = True

        m = re.compile(r'票 票:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['recommend_count'] = int(m.group('T'))

        m = re.compile(r'内容简介:(?P<T>.*/>)(?P<Name>.*?)最新章节',
                       re.S).search(info.prettify())
        if m:
            a['desc'] = m.group('T').strip()  #md.convert
            a['name'] = m.group('Name').strip()

        #print a
        return a

    except urllib2.HTTPError:
        log("Error: book_id " + str(id) + " not exists.")
    except Exception, ex:
        log("Error: book_id " + str(id) + " " + str(Exception) + ":" + str(ex))
Exemplo n.º 37
0
def msgHandler(msg):
    msgTag = ''
    msgAtt = ''
    msgCmd = ''
    result = ''
    message = msg.split('/')
    log(message)
    if len(message) == 1:
        msgTag = message[0]
    if len(message) == 2:
        msgTag = message[0]
        msgAtt = message[1]
    if len(message) == 3:
        msgTag = message[0]
        msgAtt = message[1]
        msgCmd = message[2]

    log(msgTag + '/' + msgAtt + '/' + msgCmd)

    for tag in tags:
        if msgTag != '':
            if tags[tag]['name'] == msgTag:
                if msgCmd == '':
                    # Request received
                    if msgAtt == 'io':
                        result = str(tags[tag][msgAtt].value())
                    else:
                        if msgAtt != '':
                            try:
                                result = str(tags[tag][msgAtt])
                            except:
                                result = 'Invalid attribute...'
                        else:
                            result = 'attribute missing...'
                else:
                    # Command received
                    if type(tags[tag][msgAtt]) is str:
                        tags[tag][msgAtt] = msgCmd
                    else:
                        if msgAtt == 'io':
                            tags[tag][msgAtt].value(int(msgCmd))
                        else:
                            tags[tag][msgAtt] = int(msgCmd)
                    result = 'Command: ' + msgTag + '/' + msgAtt + '/' + msgCmd + ' received!'

    if len(msg) == 0:
        result = 'connected!'
    if result != '':
        log(result)
        clientWrite(result)
    if result == '':
        log('cannot process request or command...')
        clientWrite('cannot process request or command...')
Exemplo n.º 38
0
def mkdesc ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item['id'])
                try:
                    desc = htmlstrip(b["description"]).strip()
                    db.update_book_desc(item['id'], desc)
                except Exception,ex:
                    log("Error: mkdesc "+ str(item['id']) + " " + str(Exception) + ":" + str(ex),"desc.error")
Exemplo n.º 39
0
def get_book_detail (id):
    try:
        _url = __url__ + "info/%d/%d.htm" % ((id/1000), id)
        html = urllib2.urlopen(_url).read()
        #html = unicode(html, __encode__, "ignore")

        soup = BeautifulSoup(html)
        info = soup.findAll(id=['In'])[0]

        #print type(info)
        #print info.__dict__
        #print info.prettify()

        a = dict()

        m = re.compile(r'作 者:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['author'] = m.group('T').strip()

        m = re.compile(r'类 别:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['cate'] = m.group('T').strip()

        m = re.compile(r'状 态:(?P<T>.*?)<', re.S).search(info.prettify())
        if m:
            tmp = m.group('T').strip()
            a['is_finish'] = False
            if tmp == "已完成":
                a['is_finish'] = True

        m = re.compile(r'票 票:(?P<T>.*?)<', re.S).search(info.prettify())
        if m: a['recommend_count'] = int(m.group('T'))

        m = re.compile(r'内容简介:(?P<T>.*/>)(?P<Name>.*?)最新章节', re.S).search(info.prettify())
        if m:
            a['desc'] = m.group('T').strip()#md.convert
            a['name'] = m.group('Name').strip()

        #print a
        return a

    except urllib2.HTTPError:
        log("Error: book_id "+ str(id) + " not exists.")
    except Exception,ex:
        log("Error: book_id "+ str(id) + " " + str(Exception) + ":" + str(ex) )
Exemplo n.º 40
0
def mkdesc():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item["id"])
                try:
                    desc = htmlstrip(b["description"]).strip()
                    db.update_book_desc(item["id"], desc)
                except Exception, ex:
                    log("Error: mkdesc " + str(item["id"]) + " " + str(Exception) + ":" + str(ex), "desc.error")
Exemplo n.º 41
0
    def on_event(self, event, extension):
        try:
            vrs = extension.variables
            vrs.keyword = event.query.split(" ")[
                0]  # store docset used keyword
            query = vrs.query = event.query.replace(vrs.keyword + " ", "", 1)

            # Is a docset selected ?
            if vrs.is_sel:
                log("EXEC SEARCH", LogType.DEBUG)
                vrs.query = query.strip()

                return RenderResultListAction([
                    ResultItems.docset_searchforin_result(
                        vrs.sel_docset, vrs.query),
                    ResultItems.docset_change_result(),
                ])

            query = event.query.replace(vrs.keyword + " ", "", 1)

            # If User types a docset ID withinz keyword
            if Docsets.docsetDict.get(query):
                log("SELECTED A DOCSET", LogType.DEBUG)
                # Select a docset
                extension.select_docset(query)
                # Update UI query string
                extension.update_ui_query_string(event, "")

                return RenderResultListAction([
                    ResultItems.docset_searchforin_result(vrs.sel_docset, ""),
                    ResultItems.docset_change_result(),
                ])

            # ELSE User would select a docset
            log("LIST DOCSETS", LogType.DEBUG)
            # Unselect docset
            extension.unselect_docset()
            # List docsets
            return RenderResultListAction(extension.get_sorted_docsets(query))
        except Exception as error:  # pylint: disable=broad-except
            log(error, LogType.ERROR)
            log(traceback.format_exc(), LogType.DEBUG)
Exemplo n.º 42
0
    def timed(*args, **kw):
        global is_start
        global nesting_level

        if not is_start:
            print()

        is_start = True
        log("{}.".format(method.__name__))
        nesting_level += 1

        start_time = time.time()
        result = method(*args, **kw)
        end_time = time.time()

        nesting_level -= 1
        log("/{} ({:0.2f} sec)".format(method.__name__, end_time - start_time))
        is_start = False

        return result
Exemplo n.º 43
0
    def __init__(self):
        """Initiate and include some class"""
        dir = os.path.dirname(__file__)
        includedir = os.path.join(dir, "../")
        sys.path.append(includedir)

        from lib.log import *
        from conf.Config import *

        self.Conf = Config()
        self.Log = log()
Exemplo n.º 44
0
def save_text (id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin '+ str(id)+'-------------'
        chs = dw.get_chapters(id)
        if chs is not None or len(chs) > 0:
            for var in chs:
                print var['id']
                text = ''
                t = dw.get_chapter_text(var['id'])
                if t is None:
                    try:
                        text = get_chapter(id, var['id'])
                        if text is not None and text != '':
                            dw.insert_chapter_text(var['id'], text)
                            print '-------------end of '+ str(id)+'-----------'
                    except Exception,ex:
                        log("Error: get_chapter "+ str(id) + " " + str(Exception)+":"+str(ex) )
                print "Info: chapter "+str(var['id'])+" exists"
        else:
            print 'Info: have no chapters'
Exemplo n.º 45
0
    def test_login(self):
        try:
            #输入账号密码
            accountInput = self.driver.find_element_by_id(
                "j-phone-normal-input")
            accountInput.send_keys(self.account)
            pswInput = self.driver.find_element_by_id(
                "j-password-normal-input")
            pswInput.send_keys(self.password)
            #输入回车
            pswInput.send_keys(Keys.RETURN)

            try:
                #输入账号密码登录后,看下错误弹窗有无出来
                WebDriverWait(self.driver, 2).until(
                    EC.visibility_of_element_located(
                        (By.CLASS_NAME, "xh-message")))
                #如果有出来,截图,并且发钉钉消息
                log('发现异常')
                self.captureAndUpload('error')
                sendDingMsg(
                    self.createDingMessageJson(title='天幕管理后台前端监控',
                                               subtitle='登录测试未通过',
                                               imgList=self.ossImgList))

            except:
                #如果没有出来,检查进入主页后有无class为org-list的元素(页面是否正常加载)
                WebDriverWait(self.driver, 2).until(
                    EC.presence_of_element_located(
                        (By.CLASS_NAME, "org-list")))

        except:
            #若报错,截图,发钉钉消息
            log('发现异常')
            self.captureAndUpload('error')
            sendDingMsg(
                self.createDingMessageJson(title='天幕管理后台前端监控',
                                           subtitle='登录测试未通过',
                                           imgList=self.ossImgList))
Exemplo n.º 46
0
    def setUp(self):
        driverPath = os.path.abspath('chromedriver')
        webaddr = webInfo['addr']

        option = webdriver.ChromeOptions()
        option.add_argument("headless")
        self.driver = Chrome(driverPath, options=option)

        self.driver.get(webaddr)
        #模拟登录账号
        self.account = webInfo['account']
        #模拟登录密码
        self.password = webInfo['password']
        #图片上传oss的地址
        self.ossPath = webInfo['ossPath']
        #图片的cdn域名
        self.cdnHost = webInfo['cdnHost']
        #若有错误,截图后图片的本地的地址
        self.localImgList = []
        #若有错误,截图并上传到oss后,oss的地址
        self.ossImgList = []
        log('网址已打开:  ' + webaddr + '。现在登录, 账号: ' + self.account + ', 密码: ' +
            self.password)
Exemplo n.º 47
0
Arquivo: ddw.py Projeto: wangjun/novel
def save_text(id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin ' + str(id) + '-------------'
        chs = dw.get_chapters(id)
        if chs is not None or len(chs) > 0:
            for var in chs:
                print var['id']
                text = ''
                t = dw.get_chapter_text(var['id'])
                if t is None:
                    try:
                        text = get_chapter(id, var['id'])
                        if text is not None and text != '':
                            dw.insert_chapter_text(var['id'], text)
                            print '-------------end of ' + str(
                                id) + '-----------'
                    except Exception, ex:
                        log("Error: get_chapter " + str(id) + " " +
                            str(Exception) + ":" + str(ex))
                print "Info: chapter " + str(var['id']) + " exists"
        else:
            print 'Info: have no chapters'
Exemplo n.º 48
0
def scheduler():
    min = (utime.localtime()[4])
    hour = int((utime.localtime()[3]) + 7)

    # Adjust hours between 24-30
    if hour > 23 and hour < 31:
        hour = hour - 24

    currTime = str(hour) + ':' + str(min)
    print(currTime)
    clientWrite('time-' + currTime)

    if hour == 23 and min == 45:

        try:
            os.remove('logs.txt')
            log('logs cleared...\n')
        except:
            log('logs file purge error!')

    if min == 58:
        lib.syncClock()

    for tag in tags:
        schedule = tags[tag]['schedule']
        sched = schedule.split('/')
        for hm in sched:
            start = hm.split(' ')[0]
            startH = int(start.split(':')[0])
            startM = int(start.split(':')[1])
            stop = hm.split(' ')[1]
            stopH = int(stop.split(':')[0])
            stopM = int(stop.split(':')[1])
            if hour == startH and min >= startM and min < stopM:
                if tags[tag]['io'].value() == 1:
                    log('start tag: ' + tags[tag]['name'])
                    tags[tag]['io'].value(0)
                    clientWrite(tags[tag]['name'] + '/0')

            if ((hour == startH) and (min >= stopM)) or ((hour == startH + 1) and (min - stopM < 0)):
                # if hour == startH:
                if tags[tag]['io'].value() == 0:
                    log('stop tag: ' + tags[tag]['name'])
                    # print(type(tag))
                    tags[tag]['io'].value(1)
                    clientWrite(tags[tag]['name'] + '/1')
Exemplo n.º 49
0
def graph(name, functions, settings, preview): # x_pos, y_pos, scale):
    x_pos, y_pos, scale = settings['x_pos'], settings['y_pos'], settings['scale']
    log("pl.graph")
    grapher.main(name, functions, x_pos, y_pos, scale / 10 * const.PIXELS_PER_MM, preview)
Exemplo n.º 50
0
# ping a list of hosts
def ping(hosts):
  for host in hosts:
    pipe = subprocess.Popen([ping_executable, '-c', str(packets), host],
                            stdout=subprocess.PIPE)
    response = pipe.communicate()[0].splitlines()[-1] # last line
    yield (host, parse_ping_response(response))

# collect data
def collect(hosts):
  if ping_method == 'fping':
    responses = fping(hosts)
  else:
    responses = ping(hosts)

  for (host, delay) in responses:
    add_value(host, delay)
    log('collecting host ' + host + ': delay ' + delay, 1)

if __name__ == "__main__":
  from lib.get_hosts import get_hosts

  log('Collect started', 1)

  tstart = time.time()
  collect(get_hosts())
  tend = time.time() - tstart

  log('Collect finished and ran for ' + str(tend), 1)
Exemplo n.º 51
0
def main():
    log("started")
    log(sys.path)
    log("wait while unlocking...")
    while is_locked():
        time.sleep(1)
    lock()
    try:
        log("generating cnc")
        gen_cnc()
        log("sending cnc")
        send_cnc()
        log("done")
    except Exception as e:
        log("ERROR")
        tb = traceback.format_exc()
        print(tb, file=sys.stderr)
    unlock()
    log("stopped")
Exemplo n.º 52
0
def unlock():
    os.remove('.lock')
    log("unlocked")
Exemplo n.º 53
0
def lock():
    open('.lock', 'a').close()
    log("locked")