Ejemplo n.º 1
0
    def start(self):
        self.running = True
        logger.info("TCP socket server [%s:%s] start." % (self.host, self.port))
        while self.running:
            selectors = self.selector.poll()
            for fd, event in selectors:
                if fd == self.server_socket.fileno():
                    sock, addr = self.server_socket.accept()

                    # new client
                    sock.setblocking(False)
                    self._set_socket_opt(sock)
                    self.clients.update({sock.fileno(): sock})
                    self.clients_ext.update({sock.fileno(): addr})
                    self.selector.register(sock.fileno(), select.EPOLLIN | select.EPOLLPRI | select.EPOLLET)
                    logger.info("new socket client: [%s]" % str(addr))

                elif event & select.EPOLLIN:
                    try:
                        recv_data = self.recv(self.clients[fd])
                        if self.handler and recv_data:
                            self.handler.handle(self, self.clients[fd], recv_data)
                    except Exception, e:
                        logger.error("recv data and handle data with exception: %s" % e)
                        if fd in self.clients_ext.keys():
                            logger.info("close socket [%s]" % str(self.clients_ext[fd]))
                            #self.send(sock, "close you!", 10)
                            self.close_socket(self.clients[fd])
                elif event & select.EPOLLPRI:
                    logger.info("EPOLLPRI")
                elif event & select.EPOLLERR or event & select.EPOLLHUP:
                    logger.info("socket [%s] is error, closed it." % str(self.clients_ext[fd]))
                    if fd in self.clients_ext.keys():
                        self.close_socket(self.clients[fd])
Ejemplo n.º 2
0
def dowload_pdf_and_convert():
    projects = db.read_data()
    for project in projects:
        if project.get("downloaded"):
            pass
        else:
            browser.get(project["link"])
            time.sleep(5)
            href = browser.find_elements_by_css_selector("tr#tile30 td.vs1 a")
            if len(href) != 1:
                log.error("web format changed, cannot find downloadlink")
                break
            else:
                href = href[0]
                pdf_file = requests.get(href.get_attribute("href"), stream=True)
                with open("..\\data\\{}.pdf".format(project.get("title")), "wb") as f:
                    for chunk in pdf_file.iter_content(chunk_size=128):
                        if chunk:
                            f.write(chunk)
                            f.flush()
                project["downloaded"] = time.strftime("%d_%m_%Y")
                log.info("{} has been downloaded.".format(project.get("title")))

                try:
                    with pdfplumber.open("..\\data\\{}.pdf".format(project.get("title"))) as pdf:
                        txt_filename = project.get("title") + ".txt"
                        with codecs.open("..\\data\\{}".format(txt_filename), "w", encoding="utf-8") as new_f:
                            for page in pdf.pages:
                                new_f.write(page.extract_text())
                    log.info("{} had been converted.".format(project.get("title")))
                except:
                    log.error("{} had convert failed.".format(project.get("title")))

    db.write_data(projects)
Ejemplo n.º 3
0
def make_order(order_num):

    order_num = str(order_num)
    ts = int(time.time())
    # step1
    r = requests.post("https://m.xiaomiyoupin.com/app/stat/visitv2",
                      data=[{"e":{"et":"shop","ref":"$Detail$?v=8&id=https%3A%2F%2Fm.xiaomiyoupin.com%2Fdetail%3Fgid%3D{order_num}&event=TOUCH&time={ts}&area=buy_confirm&iid=g%3D{order_num}%26pid%3D87152".format(order_num=order_num, ts=ts),
                                  "t":ts,"spm":"YouPinM.$Detail$_{order_num}.buy_confirm.0.50164756".format(order_num=order_num)}}],
                      headers={
                          "Content-Type":"application/x-www-form-urlencoded",
                          "DToken":"",
                          "Origin":"https://m.xiaomiyoupin.com",
                          "Referer":"https://m.xiaomiyoupin.com/detail?gid=115147",
                          "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Mobile Safari/537.36",
                          "X-User-Agent":"channel/youpin platform/youpin.m",
                          "X-Yp-App-Source":"front-RNWeb-old"
                      })
    log.info("step1 result: "+str(r.status_code))

    # step2
    r = requests.post("https://m.xiaomiyoupin.com/api/auth/login/isloggedin",
                      headers={ # Reffer 中有加密字段, 暂时放弃
                          "Content-Type":"application/x-www-form-urlencoded",
                          "DToken":"",
                          "Origin":"https://m.xiaomiyoupin.com",
                          "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Mobile Safari/537.36",
                          "X-User-Agent":"channel/youpin platform/youpin.m",
                          "X-Yp-App-Source":"front-RNWeb-old"
                      })
    log.info("step2 result: "+str(r.text))

    # step3
    r = requests.post("https://m.xiaomiyoupin.com/app/stat/visitv2",
                      )
Ejemplo n.º 4
0
    def get(self):
        page = int(self.get_argument("page", 1))
        q = self.get_argument('q', '')
        if not q :
            self.redirect('/images_info')
            return
        sql = 'SELECT * FROM tb_images WHERE name LIKE upper(?) OR tags LIKE upper(?) OR info LIKE upper(?) OR author LIKE upper(?) OR types LIKE upper(?) ;'
        images_count = self.db_select(sql, ['%' + q + '%', '%' + q + '%', '%' + q + '%', '%' + q + '%', '%' + q + '%'])


        sql = 'SELECT * FROM tb_images WHERE name LIKE upper(?) OR tags LIKE upper(?) OR info LIKE upper(?) OR author LIKE upper(?) OR types LIKE upper(?) LIMIT ?,?;'
        images_result = self.db_select(sql, ['%' + q + '%', '%' + q + '%', '%' + q + '%', '%' + q + '%', '%' + q + '%',  (page - 1) * page_size, page_size])

        sql = 'SELECT images_id FROM tb_status WHERE containers_user = ? AND containers_status = "runing";'
        statrt_result = self.db_select(sql, [self.current_user.decode()])

        result = []
        for x in images_result:
            if {'images_id': x['images_id']} not in statrt_result:
                x['json_images_port'] = json.loads(x['images_port'])
                result.append(x)

        result_count = []

        for x in images_count:
            if {'images_id': x['images_id']} not in statrt_result:
                result_count.append(x)

        logger.info('获取用户%s搜索的可以使用的镜像!' % self.current_user.decode())
        self.render('images.html', cursor = result, count = len(result_count))
Ejemplo n.º 5
0
    def get(self):
        images_id = self.get_argument("images_id", '').strip()

        sql = 'SELECT * FROM tb_images WHERE images_id = ? ;'
        result = self.db_select(sql, [images_id])
        if not result :
            logger.error('用户%s提交的镜像id错误' % self.current_user.decode())
            self.render('docker.html', status = '', port = '', error = True)
            return

        port, containers_id, tf = self.__start(result[0]['images_id'], result[0]['images_port'])

        if not containers_id:
            logger.error('用户%s开启容器时错误,错误镜像id是%s' % (self.current_user.decode(), result[0]['images_id']))
            self.render('docker.html', status = '', port = '', error = True)
            return

        if not tf :
            r = self.__insert_sql(containers_id, result[0]['images_id'], json.dumps(port))
            if r :
                logger.info('用户%s开启容器成功,容器id是%s,数据写入数据库成功!' % (self.current_user.decode(), containers_id))
            else:
                logger.error('用户%s开启容器成功,容器id是%s,数据写入数据库失败!' % (self.current_user.decode(), containers_id))
                con = Dockers_Info.Get_Containers_Message(containers_id)
                Dockers_Stop.Stop_Containers(con)
                self.render('docker.html', status = '', port = '', error = True)
                return

        self.render('docker.html', status = 'start', port = self.__get_http(port), error = False)
Ejemplo n.º 6
0
    def __re_port(self, strings):

        result = []
        logger.info('处理传入的环境端口信息!')

        string = re.split(r',|;', strings.lower())


        if not string:
            logger.error('传入的环境端口信息有误!错误端口信息为:%s' % strings)
            return ''

        res = r'(\d{1,5})/(tcp|udp)'

        print(string)
        for _ in string:
            if _ == '':
                continue
            r = re.search(res, _)

            if not r :
                logger.error('传入的环境端口信息有误!错误端口信息为:%s' % _)
                return ''

            port = r.group(1)
            protocol = r.group(2)
            result.append({'port': port, 'protocol': protocol})

        logger.info('成功处理传入的环境端口信息,获取到端口信息为:%s' % json.dumps(result))
        return json.dumps(result)
Ejemplo n.º 7
0
def check_all_shops():
    with open(os.path.join(os.path.dirname(__file__),"config","shop.json"), "r", encoding='UTF-8') as f:
        infos = json.loads(f.read())
        for info in infos:
            for shop in info["shop"]:
                log.info("checking {} / {}".format(shop, info.get("keyword")))
                keywords = info.get("key_word").split(",")
                check_shop(shop, keywords)
Ejemplo n.º 8
0
 def getZhuanXianPages(self, pages=pages):
     city_list = self.getZhuanXianCityList()
     logger.info(city_list)
     for city in random.shuffle(city_list):
         for page in range(1, pages):
             for url in self.getUrlsForCity(city, page):
                 for page in range(1, pages):
                     for id in self.getIdsForAUrl(url, page):
                         self.getAndSaveZhuanXianFromId(id)
Ejemplo n.º 9
0
 def __get_port(self):
     logger.info('获取系统已经开启的端口,并返回一个随机端口')
     cmd = "netstat -ntl | grep -v Active | grep -v Proto | awk '{print $4}' | awk -F: '{print $NF}'"
     ports = os.popen(cmd).read().split('\n')
     port = random.randint(1024, 65535)
     if str(port) not in ports:
         return port
     else:
         self.__get_port()
Ejemplo n.º 10
0
    def stop(self):
        self.running = False
        logger.info("TCP socket server [%s:%s] stop." % (self.host, self.port))

        for fd, client in self.clients.items():
            self.selector.unregister(fd)
            self.clients_ext.pop(fd)
            client.close()
        self.server_socket.close()
        self.selector.close()
Ejemplo n.º 11
0
    def stop(self):
        self.running = False
        logger.info("TCP socket server [%s:%s] stop." % (self.host, self.port))

        for fd,client in self.clients.items():
            self.selector.unregister(fd)
            self.clients_ext.pop(fd)
            client.close()
        self.server_socket.close()
        self.selector.close()
Ejemplo n.º 12
0
 def __send_messages(cls, que):
     asyncio.set_event_loop(asyncio.new_event_loop())
     while 1:
         status = json.dumps(que.get())
         for waiters in cls.waiters:
             try:
                 waiters.write_message(status)
                 logger.info('向客户端发送系统信息成功!')
             except Exception as e:
                 continue
         que.queue.clear()
Ejemplo n.º 13
0
    def get(self):
        sql = 'SELECT containers_id FROM tb_status WHERE containers_user = ? and containers_status = ?;'
        containers_list = self.db_select(sql, [self.current_user.decode(), 'runing'])

        if containers_list:
            logger.error('用户%s没有关闭所有的容器,无法退出登陆!' % self.current_user.decode())
            self.render('logout.html')
            return

        logger.info('用户%s退出登录!' % self.current_user.decode())
        self.clear_cookie("cookie_user")
        self.redirect("/")
Ejemplo n.º 14
0
    def __re_hub(self, hub):

        result = hub.strip().split(':')
        if len(result) > 1 :
            images_name = result[0]
            images_tag = result[1]
        else:
            images_name = hub.strip()
            images_tag = 'latest'
        if images_name :
            logger.info('成功获取到镜像名称为:%s:%s' % (images_name, images_tag))
        return images_name, images_tag
Ejemplo n.º 15
0
    def get(self):
        sql = 'SELECT * FROM tb_status WHERE containers_user = ? AND containers_status = "runing";'
        status_result = self.db_select(sql, [self.current_user.decode()])
        images_result = self.db_select('SELECT * FROM tb_images;')

        for _ in status_result:
            sql = 'SELECT * FROM tb_images WHERE images_id = ?'
            _['images_info'] = self.db_select(sql, [_['images_id']])[0]
            _['json_containers_port'] = json.loads(_['containers_port'])


        logger.info('获取用户%s已开启的镜像名字和端口' % self.current_user.decode())
        self.render('status.html', sysinfo = self.status, cursor = status_result, start_counts = len(status_result), images_counts = len(images_result))
Ejemplo n.º 16
0
async def post_json(url, params, semaphore_num=500):
    logger.info(f"async_post_json url is {url} params are {params}")
    semaphore = asyncio.Semaphore(semaphore_num)
    async with semaphore:
        async with ClientSession() as session:
            try:
                async with session.post(url, json=params) as resp:
                    try:
                        return await resp.json()
                    except ContentTypeError as e:
                        logger.exception(f'Result is not json formatted, {e}')
                        return  # await resp.text()
            except asyncio.TimeoutError as e:
                logger.exception(f'current request timeout. params is {params}')
Ejemplo n.º 17
0
 def getAndSaveZXFromId(self, id):
     url = "http://56gate.com/html/zx/zxdetail_%s.html" % id
     try:
         GateZhuanXian.objects.get(webSiteId=id)
         logger.info("Zhuanxian %s already exsists" % id)
         return
     except:
         pass
     text = self.httpClient.geturlcon(url)
     result = {}
     try:
         result = self.phaseLine(text, id)
     except Exception, e:
         logger.error(e)
Ejemplo n.º 18
0
 def getAndSaveHYFromId(self, id):
     url = "http://www.tk56.com/goodsdetail.aspx?id=%s&t=1" % id
     try:
         TK56HuoYuan.objects.get(webSiteId=id)
         logger.info("%s already exsists" % id)
         return
     except:
         pass
     text = self.httpClient.geturlcon(url)
     result = {}
     try:
         result = self.pharseHYMeta(text, id)
     except Exception, e:
         logger.info(traceback.format_exc())
Ejemplo n.º 19
0
 def getAndSaveZhuanXianFromId(self, id):
     url = "http://www.chinawutong.com/201/%s.html" % id
     result = {}
     try:
         WTZhuanXian.objects.get(webSiteId=id)
         logger.info("%s already exsists" % id)
         return
     except:
         pass
     text = self.httpClient.geturlcon(url)
     try:
         result = self.pharseZhuanXianMeta(text, id)
     except Exception, e:
         logger.error(traceback.format_exc())
Ejemplo n.º 20
0
    def post(self):
        logger.info('提交数据进行修改密码')
        old_password = self.get_argument("old_password", '')
        new_password = self.get_argument("new_password", '')
        if not old_password.strip() or not new_password.strip():
            logger.info('密码不能为空')
            self.render('change_pass.html', error = 1)
            return

        md = hashlib.md5()
        md.update(old_password.encode('utf-8'))
        sql = 'SELECT password FROM tb_userinfo WHERE username = ? LIMIT 1;'
        result = self.db_select(sql, [self.current_user.decode()])

        if md.hexdigest() == result[0]['password']:
            md = hashlib.md5()
            md.update(new_password.encode('utf-8'))
            sql = 'UPDATE tb_userinfo SET password = ? WHERE username = ?'
            result = self.db_update_insert(sql, [md.hexdigest(), self.current_user.decode()])
            if not result:
                logger.warning('数据库更新失败,密码更改失败')
                self.render('change_pass.html', error = 2)
                return
        else :
            logger.info('原来密码错误')
            self.render('change_pass.html', error = 3)
            return
        logger.info('密码修改成功!')
        self.render('change_pass.html', error = 0)
Ejemplo n.º 21
0
def read_data():
    data = []
    try:
        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)),"data.json"), "r") as f:
            try:
                data = json.load(f)
            except json.decoder.JSONDecodeError:
                log.info("data file is empty or cannot decode, reseted.")
                pass
    except FileNotFoundError:
        log.info("data file not exist, created one.")
        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)),"data.json"), "w") as f:
            pass
    return data
Ejemplo n.º 22
0
 def get(self):
     containers_id = self.get_argument("containers_id", '').strip()
     if not containers_id:
         logger.error('用户%s提交的容器id错误' % self.current_user.decode())
         self.render('docker.html', status = '', port = '', error = True)
         return
     logger.info('用户%s提交的容器id是%s。' % (self.current_user.decode(),containers_id))
     con = Dockers_Info.Get_Containers_Message(containers_id)
     if not con:
         logger.error('用户%s提交的容器id错误' % self.current_user.decode())
         self.render('docker.html', status = '', port = '', error = True)
         return
     logger.info('获取到用户%s提交的容器信息。' % self.current_user.decode())
     if not Dockers_Stop.Stop_Containers(con):
         logger.error('获取用户%s提交的容器信息,但是在关闭的时候出错。' % self.current_user.decode())
         self.render('docker.html', status = '', port = '', error = True)
         return
     logger.info('成功关闭用户%s提交的容器。' % self.current_user.decode())
     sql = "UPDATE tb_status SET containers_status = 'closed' WHERE containers_id = ? ;"
     result = self.db_update_insert(sql, [containers_id])
     if not result:
         logger.error('成功关闭用户%s提交的容器,但是在写入数据库时出错。' % self.current_user.decode())
         self.render('docker.html', status = '', port = '', error = True)
         return
     logger.info('成功关闭用户%s提交的容器,并成功写入数据库。' % self.current_user.decode())
     self.render('docker.html', status='close', port='', error = False)
Ejemplo n.º 23
0
    def __init__(self):
        '''
        初始化tornado
        '''

        # tornado的路由信息
        handlers = [
            (r"/", HomeHandler),
            (r"/login", LoginHandler),

            (r"/logout", LogoutHandler),
            (r"/stop_all_containers", StopAllContainers),

            (r"/search_images", SearchImagesHandler),
            (r"/images_info", ImagesHandler),
            (r"/status_info", StatusHandler),
            (r"/websocket", SocketHandler),
            (r"/setting", SettingHandler),


            (r"/add_images", AddImagesHandler),
            (r"/start_containers", StartContainersHandler),
            (r"/stop_containers", StopContainersHandler),

            (r"/change_pass", Change_Pass_Handler),
            (r"/add_user", Add_User_Handler),

            (r".*", ErrorHandler),
        ]
        # 初始化tornado的设置
        settings = dict(
            template_path = os.path.join(os.path.dirname(__file__), "templates"),
            static_path = os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies = False,
            cookie_secret = "__TODO:_TORNADO_MY_OWN_RANDOM_VALUE_HERE__",
            login_url = "/login",
            debug = True,
        )

        logger.info('获取tronado基础配置')
        tornado.web.Application.__init__(self, handlers, **settings)
        logger.info('初始化tornado对象,初始化路由')
        self.db = sqlite3.connect(options.sqlite_path, check_same_thread = False)
        self.db.row_factory = self.__dict_factory
        logger.info('链接数据库')
        threading.Thread(target = Start_Get_Sysinfo, args = (que,)).start()
        logger.info('开启后台监控进程')
        self.status = que.get()
        logger.info('获取系统信息!')
Ejemplo n.º 24
0
    def getAndSaveCYFromId(self, id):
        url = "http://56gate.com/html/cy/cydetail_%s.html" % id
        try:
            GateCYSource.objects.get(webSiteId=id)
            logger.info("Zhuanxian %s already exsists" % id)
            return
        except:
            pass

        text = self.httpClient.geturlcon(url)
        result = {}
        try:
            result = self.parseCYDetailsMeta(text, id)
        except Exception, e:
            logger.error(e)
Ejemplo n.º 25
0
 def send(self, sock, data, size):
     if size != len(data):
         logger.error("data size [%d] not equal real size [%d]", size, len(data))
         return -1
     logger.info("send-DATA: %s" % data)
     msg = DataMsg(data)
     buf = msg.pack()
     try:
         ret = sock.send(buf)
         return ret - SOCKET_HEADER_SIZE
     except socket.error as err:
         logger.error("socket exception: %s" % err)
         if err.errno == errno.ECONNRESET or err.errno == errno.EBADF:
             logger.info("client [%s:%s] is closed." % self.clients_ext[sock.fileno()])
             self.close_socket(sock)
Ejemplo n.º 26
0
def recode_name(name):
    if not have_chinese_(name):
        try:
            new_name = name.encode('cp437').decode('utf-8')
        except Exception as error:
            logger.info(f'当前编码方式不正确,错误信息为:{error}')
            try:
                new_name = name.encode('cp437').decode('gbk')
            except Exception as error:
                logger.info(f'当前编码方式不正确,错误信息为:{error}')
                new_name = name
    else:
        new_name = name

    return new_name
Ejemplo n.º 27
0
    def getAndSaveCYFromId(self, id):
        url = "http://wl.kywmall.com/wl_pages/wl_find_truck_details.aspx?id=%s" % id
        try:
            KYVehicle.objects.get(webSiteId=id)
            logger.info("HuoYuan %s already exsists" % id)
            return
        except:
            pass
        text = self.httpClient.geturlcon(url)

        result = {}
        try:
            result = self.parseCheyuanMeta(text, id)
        except Exception, e:
            logger.error(e)
Ejemplo n.º 28
0
    def getAndSaveFromId(self, id):
        url = "http://land.jctrans.com/CarsInfo/roaddetails_%s.html" % id
        try:
            JctransRoadDetails.objects.get(webSiteId=id)
            logger.info("%s already exsists" % id)
            return
        except:
            pass

        result = {}
        text = self.httpClient.geturlcon(url)
        try:
            result = self.parseRoadDetailsMeta(text, id)
        except Exception, e:
            logger.error(e)
Ejemplo n.º 29
0
    def getAndSaveZXFromId(self, id):
        url = "http://wl.kywmall.com/wl_pages/wl_find_line_details.aspx?id=%s" % id
        try:
            KYZhuanXian.objects.get(webSiteId=id)
            logger.info("Zhuanxian %s already exsists" % id)
            return
        except:
            pass
        text = self.httpClient.geturlcon(url)

        result = {}
        try:
            result = self.parseZXMeta(text, id)
        except Exception, e:
            logger.error(e)
Ejemplo n.º 30
0
    def getAndSaveHYFromId(self, id):
        url = "http://56gate.com/html/hy/sgdetail_%s.html" % id
        try:
            GateHuoYuanSource.objects.get(webSiteId=id)
            logger.info("HuoYuan %s already exsists" % id)
            return
        except:
            pass

        text = self.httpClient.geturlcon(url)
        result = {}
        try:
            result = self.parseHYDetailsMeta(text, id)
        except Exception, e:
            traceback.print_exc()
            logger.error(e)
Ejemplo n.º 31
0
 def geturlcon(self, url, data=None, headers={}, timeout=20, retries=5):
     logger.info(url)
     tries = 0
     while tries < retries:
         try:
             request = urllib2.Request(url, data=self.getPostData(data), headers=headers)
             url = urllib2.urlopen(request, timeout=timeout)
             time.sleep(1)
             #page = url.read().decode('utf8', 'ignore')
             page = url.read()
             logger.warn(page)
             return page
         except Exception, e:
             print traceback.format_exc()
             tries += 1
             time.sleep(5 * random.random() + 5)
Ejemplo n.º 32
0
def send_mail(subject, content, receiver):
    message = MIMEText(content, 'plain', 'utf-8')
    message['From'] = Header(mail_config["user"], 'utf-8')
    message['To'] = Header(config.email, 'utf-8')
    message['Subject'] = Header(subject, 'utf-8')

    try:
        smtpObj = smtplib.SMTP()
        smtpObj.connect(mail_config["host"], 25)  # 25 为 SMTP 端口号
        smtpObj.login(mail_config["user"], mail_config["passwd"])
        smtpObj.sendmail(mail_config["user"], receiver, message.as_string())
        log.info("mail sent success.")
    except smtplib.SMTPException as e:
        log.error("mail sent failed")
        log.error(str(e))
        log.error(traceback.format_exc())
Ejemplo n.º 33
0
    def __re_risk(self, string):

        if not string.strip():
            return ''

        string = string.strip().lower()
        logger.info('处理传入的环境难度等级信息!')

        lists = ['simple', 'medium', 'advanced']

        if string not in lists:
            logger.error('传入的环境难度等级信息有误!')
            return ''

        logger.info('成功处理传入的环境难度等级信息,获取到难度等级为:%s' % string)
        return string
Ejemplo n.º 34
0
 def send(self, sock, data, size):
     if size != len(data):
         logger.error("data size [%d] not equal real size [%d]", size,
                      len(data))
         return -1
     logger.info("send-DATA: %s" % data)
     msg = DataMsg(data)
     buf = msg.pack()
     try:
         ret = sock.send(buf)
         return ret - SOCKET_HEADER_SIZE
     except socket.error as err:
         logger.error("socket exception: %s" % err)
         if err.errno == errno.ECONNRESET or err.errno == errno.EBADF:
             logger.info("client [%s:%s] is closed." %
                         self.clients_ext[sock.fileno()])
             self.close_socket(sock)
Ejemplo n.º 35
0
    def get(self):

        logger.info('用户%s退出登陆并尝试关闭所有的容器!' % self.current_user.decode())
        sql = 'SELECT containers_id FROM tb_status WHERE containers_user = ? and containers_status = ?;'
        containers_list = self.db_select(sql, [self.current_user.decode(), 'runing'])

        for x in containers_list:
            con = Dockers_Info.Get_Containers_Message(x['containers_id'])
            if not con:
                continue
            Dockers_Stop.Stop_Containers(con)


        sql = 'UPDATE tb_status SET containers_status = ? WHERE containers_user = ? AND containers_status = ?;'
        self.db_update_insert(sql, ['closed', self.current_user.decode(), 'runing'])
        logger.info('关闭用户%s所有的容器,并更新数据库状态!' % self.current_user.decode())
        self.redirect("/logout")
Ejemplo n.º 36
0
    def run(self):
        board = Board()
        is_piece_picked = False
        piece_src_position = None

        while True:
            for event in pygame.event.get():
                if event.type == MOUSEBUTTONUP:
                    button_up_pos = pygame.mouse.get_pos()
                    if not is_piece_picked:
                        which_piece_is_picked = self.getPieceByPosition(
                            board, button_up_pos)
                        if which_piece_is_picked != 0 and board.isTurnRight(
                                self.toBoardPos(button_up_pos)):
                            is_piece_picked = True
                            piece_src_position = button_up_pos
                            logger.info("pick an chess %s, src pos is %s",
                                        str(which_piece_is_picked),
                                        str(piece_src_position))
                    else:
                        src = self.toBoardPos(piece_src_position)
                        dst = self.toBoardPos(button_up_pos)
                        another_pick = self.getPieceByPosition(
                            board, button_up_pos)

                        # pick another self chess
                        if another_pick != 0 and board.isSameSide(src, dst):
                            which_piece_is_picked = another_pick
                            piece_src_position = button_up_pos
                            logger.info("pick another chess %s, src pos is %s",
                                        str(which_piece_is_picked),
                                        str(piece_src_position))
                            break
                        else:
                            isValid, isFinished = board.move(
                                which_piece_is_picked, src, dst)
                            if isValid:
                                is_piece_picked = False
                                piece_src_position = None
                            else:
                                logger.info('invalid move src %s, dst %s',
                                            str(src), str(dst))

                            if isFinished:
                                logger.info('finished.')

                if event.type == QUIT:
                    exit()

                self.__screen.blit(self.__background, (0, 0))
                for c in board.getPieces():
                    self.putPiece(c[0], c[1])
                pygame.display.update()
Ejemplo n.º 37
0
def unpack_zip(zipfilename='', path_from_local=''):
    index, ext = arg_first(
        ZIP_FORMAT, lambda n: n.lower() == zipfilename[-len(n):].lower())
    # check if the file format is need unpack

    if index is None:
        return

    filepath = os.path.join(path_from_local, zipfilename)
    extract_path = filepath[:-len(ext)] + '/'

    if not os.path.exists(extract_path):
        os.makedirs(extract_path)
    # else:
    #     shutil.rmtree(extract_path, True)
    #     os.makedirs(extract_path)

    patoolib.extract_archive(filepath, verbosity=0, outdir=extract_path)

    file_name, full_name = os.path.split(filepath)
    filename, ext = os.path.splitext(full_name)

    file_dir = os.path.join(root, file_name, filename)
    # 先修正文件夹名称乱码
    fix_folder_name(file_dir)
    # 修正文件名乱码
    fix_file_name(file_dir)

    # 解压后的文件列表
    name_list = []
    for name in get_all_files(extract_path):
        name_list.append(recode_name(name))

    for name in name_list:
        try:
            new_zip_filepath = os.path.join(extract_path, name)
            unpack_zip(zipfilename=new_zip_filepath)
        except NotImplementedError as e:
            logger.info(e)
        except OSError as e:
            logger.info(e)
    return extract_path
Ejemplo n.º 38
0
    def get(self):
        page = int(self.get_argument("page", 1))
        sql = 'SELECT * FROM tb_images LIMIT ?,?;'
        images_result = self.db_select(sql, [(page - 1) * page_size, page_size])

        images_count = self.db_select('SELECT id FROM tb_images')

        sql = 'SELECT images_id FROM tb_status WHERE containers_user = ? AND containers_status = "runing";'
        statrt_result = self.db_select(sql, [self.current_user.decode()])

        result = []

        for x in images_result:
            if {'images_id': x['images_id']} not in statrt_result:
                x['json_images_port'] = json.loads(x['images_port'])
                result.append(x)


        logger.info('获取用户%s可以使用的镜像!' % self.current_user.decode())
        self.render('images.html', cursor = result, count = len(images_count) - len(statrt_result) if (len(images_count) - len(statrt_result)) > 0 else 0)
Ejemplo n.º 39
0
    def start(self):
        self.running = True
        logger.info("TCP socket server [%s:%s] start." %
                    (self.host, self.port))
        while self.running:
            selectors = self.selector.poll()
            for fd, event in selectors:
                if fd == self.server_socket.fileno():
                    sock, addr = self.server_socket.accept()

                    # new client
                    sock.setblocking(False)
                    self._set_socket_opt(sock)
                    self.clients.update({sock.fileno(): sock})
                    self.clients_ext.update({sock.fileno(): addr})
                    self.selector.register(
                        sock.fileno(),
                        select.EPOLLIN | select.EPOLLPRI | select.EPOLLET)
                    logger.info("new socket client: [%s]" % str(addr))

                elif event & select.EPOLLIN:
                    try:
                        recv_data = self.recv(self.clients[fd])
                        if self.handler and recv_data:
                            self.handler.handle(self, self.clients[fd],
                                                recv_data)
                    except Exception, e:
                        logger.error(
                            "recv data and handle data with exception: %s" % e)
                        if fd in self.clients_ext.keys():
                            logger.info("close socket [%s]" %
                                        str(self.clients_ext[fd]))
                            #self.send(sock, "close you!", 10)
                            self.close_socket(self.clients[fd])
                elif event & select.EPOLLPRI:
                    logger.info("EPOLLPRI")
                elif event & select.EPOLLERR or event & select.EPOLLHUP:
                    logger.info("socket [%s] is error, closed it." %
                                str(self.clients_ext[fd]))
                    if fd in self.clients_ext.keys():
                        self.close_socket(self.clients[fd])
Ejemplo n.º 40
0
    def __thread_addimages(self, data, images_name, images_tag, db_update_insert):

        logger.info('开启一个线程去创建images镜像,镜像地址为:%s:%s' % (images_name, images_tag))
        images = Dockers_Start.Add_Images(images_name, images_tag)
        if not images:
            return False

        images_name = images.tags[0]
        images_id = images.id

        data.append(images_name)
        data.append(images_id)

        sql = 'DELETE FROM tb_images WHERE images_name = ? ;'

        if db_update_insert(sql, [images_name,]):
            logger.info('成功创建一个images镜像并删除数据库中旧的数据!')
        else:
            logger.error('成功创建一个images镜像,但是在删除数据库中旧的数据时出错。')

        sql = 'INSERT INTO tb_images (name, info, isupload, types, tags, difficulty, author, images_port, flag, images_start_mode, images_name, images_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);'

        if db_update_insert(sql, data):
            logger.info('成功把images镜像信息写入数据库,镜像名字为:%s' % images_name)
        else:
            logger.error('把images镜像信息写入数据库时出错,镜像名字为:%s' % images_name)
Ejemplo n.º 41
0
def _get_token_(url, file_path):
    _, ext = os.path.splitext(file_path)
    tmp_file_name = get_tmp_file_path(str(uuid.uuid1()) + ext.lower())

    shutil.copy(file_path, tmp_file_name)

    assert filecmp.cmp(file_path, tmp_file_name) is True

    token = []
    with open(tmp_file_name, 'rb') as f:
        logger.info(
            f"call get tokens from document service started. url is {url} ")
        response = requests.post(url, files={'file': f})

        logger.info(
            f"call get tokens from document service ended. result status code is {response.status_code}")

        if response.status_code == 200:
            result = json.loads(response.text)
            token = result['text_tokens']
            if isinstance(result, str):
                logger.exception(f"{tmp_file_name}文件可能损坏,请将订单文件另存为正确的格式后,重新上传。")
        else:
            logger.warning(response.status_code)
            logger.info(f"get tokens from windows server error. {response.content}")
            # os.remove(tmp_file_name)

    # os.remove(tmp_file_name)

    return token
Ejemplo n.º 42
0
def get_all_info():
    '''
    动态ajax, 用浏览器获取数据比较方便
    :return: 
    '''
    current_projects = db.read_data()
    projects = []
    browser.get("http://kcb.sse.com.cn/renewal/")
    work_flag = True
    while work_flag:
        time.sleep(5) # 等待页面加载
        hrefs = browser.find_elements_by_css_selector("#dataList1_container>tbody>tr:not(:first-child) td:nth-child(2)>a")
        for href in hrefs:
            title = href.text.replace("<br>","").replace("\n","")
            if len(current_projects) and title == current_projects[0]["title"]:
                work_flag = False
                break
            else:
                projects.append({
                    "title": title,
                    "link": href.get_attribute("href")
                })
        if work_flag:
            try:
                browser.find_element_by_css_selector("a.paging_next").click()
            except selenium.common.exceptions.NoSuchElementException:
                work_flag = False
                log.info("all data loaded!")
                break
        log.info("reading projects...")
        log.info(projects)
    current_projects = projects + current_projects # 保证数据库中第一个数据是最新的.
    db.write_data(current_projects)
Ejemplo n.º 43
0
    def compute(self, node, level):
        if level == 0:
            return

        board = node.getBoard()
        movements = board.nextPossibleMovements()

        if len(movements) == 0:
            logger.info("board.nextPossibleMovements return empty list.")
            return

        for mov in movements:
            row_src, col_src, row_dst, col_dst = mov

            board_copy = board.clone()
            board_copy.move(board_copy.get(row_src, col_src),
                            (row_src, col_src), (row_dst, col_dst))
            child = TreeNode()
            child.setBoard(board_copy)
            node.addChild(child)

            next_level = level - 1
            self.compute(child, next_level)
Ejemplo n.º 44
0
            logger.info("%s already exsists" % id)
            return
        except:
            pass
        text = self.httpClient.geturlcon(url)
        result = {}
        try:
            result = self.pharseHYMeta(text, id)
        except Exception, e:
            logger.info(traceback.format_exc())
        if (result.get("startPlace")):
            source = TK56HuoYuan()
            try:
                source.save_from_a_source(result)
            except Exception, e:
                logger.info(traceback.format_exc())
            else:
                logger.info("saved")
        else:
            logger.error(url + "is a null page")


    def getPage(self, type, page):
        url = "http://www.tk56.com/search%s.aspx?&page=%s" % (type, page)
        text = self.httpClient.geturlcon(url)
        soup = BeautifulSoup(text)
        soup.prettify()
        hrefs = soup.findAll(name="a", href=re.compile(type + 'detail'))
        id_list = []
        for href in hrefs:
            str_url = str(href).split('"')
Ejemplo n.º 45
0
            return
        except:
            pass
        text = self.httpClient.geturlcon(url)
        try:
            result = self.pharseHYMeta(text, id)
        except Exception, e:
            logger.info(traceback.format_exc())
        if (result.get("startPlace")) and (result.get("destPlace")):
            huoyuan = WTHuoYuan()
            try:
                huoyuan.save_from_dic(result)
            except Exception, e:
                logger.error(traceback.format_exc())
            else:
                logger.info("saved %s" % (id))
        else:
            logger.info(url + " is a null page")


    def getAndSaveZhuanXianFromId(self, id):
        url = "http://www.chinawutong.com/201/%s.html" % id
        result = {}
        try:
            WTZhuanXian.objects.get(webSiteId=id)
            logger.info("%s already exsists" % id)
            return
        except:
            pass
        text = self.httpClient.geturlcon(url)
        try:
Ejemplo n.º 46
0
            pass

        result = {}
        text = self.httpClient.geturlcon(url)
        try:
            result = self.parseRoadDetailsMeta(text, id)
        except Exception, e:
            logger.error(e)
        if ( result.get("startPlace")) and (result.get("phone")):
            jctranRoadDetails = JctransRoadDetails()
            try:
                jctranRoadDetails.save_from_dic(result)
            except Exception, e:
                logger.error(e)
        else:
            logger.info(url + "  is a null page")

    def getPage(self, page):
        url = "http://land.jctrans.com/carsinfo/newlist-0-------1---------40-%s.html" % page

        text = self.httpClient.geturlcon(url)
        soup = BeautifulSoup(text)
        soup.prettify()
        hrefs = soup.findAll(name="a", href=re.compile('roaddetails'))
        id_list = []
        for href in hrefs:
            str_url = str(href).split('"')
            id_list.append(str_url[1].split("_")[1].replace(".html", ""))
        return id_list

    def getRoadDetailPages(self, pages=pages):
Ejemplo n.º 47
0
            return
        except:
            pass

        text = self.httpClient.geturlcon(url)
        result = {}
        try:
            result = self.parseHYDetailsMeta(text, id)
        except Exception, e:
            traceback.print_exc()
            logger.error(e)
        if True:
            source = GateHuoYuanSource()
            try:
                for k, v in result.iteritems():
                    logger.info("gateee " + str(k) + " " + str(v))
                source.save_from_a_source(result)
            except Exception, e:
                traceback.print_exc()
                logger.error(e)
            else:
                logger.info("%s saved" % (id))
        else:
            logger.error(url + " is a null page")

    def getAndSaveCYFromId(self, id):
        url = "http://56gate.com/html/cy/cydetail_%s.html" % id
        try:
            GateCYSource.objects.get(webSiteId=id)
            logger.info("Zhuanxian %s already exsists" % id)
            return
Ejemplo n.º 48
0
 def recv(self, sock, timeout=5):
     end_time = time.time() + timeout
     head = sock.recv(SOCKET_HEADER_SIZE)
     if head is None or len(head) == 0:
         logger.info("client [%s] is closed." % str(self.clients_ext[sock.fileno()]))
         self.close_socket(sock)
         return None
     header_data = struct.unpack("=%dsi" % len(SOCKET_HEADER_NAME), head[:SOCKET_HEADER_SIZE])
     logger.info("read-HEAD: %s" % str(header_data))
     header = HeaderMsg(header_data[0], header_data[1])
     total_size = data_size = header.size - SOCKET_HEADER_SIZE
     data = ""
     count = 0
     while end_time > time.time():
         try:
             buf = sock.recv(data_size)
             if buf is None or len(buf) == 0:
                 logger.info("client [%s] is closed." % str(self.clients_ext[sock.fileno()]))
                 self.close_socket(sock)
                 return None
             data = data + buf
             if len(buf) < data_size:
                 data_size = data_size - len(buf)
                 time.sleep(0.1)
                 continue
             if total_size == len(data):
                 logger.info("read-DATA len(%d)" % (len(data)))
                 return data
             if count > 20:
                 logger.error("read socket timeout.")
                 return None
         except socket.error as err:
             logger.error("socket exception: %s" % str(err))
             if err.errno == errno.ECONNRESET or err.errno == errno.EBADF:
                 logger.info("client [%s] is closed." % str(self.clients_ext[sock.fileno()]))
                 self.close_socket(sock)
                 return None
             if err.errno == errno.EAGAIN:
                 logger.info("data is coming")
                 count += 1
                 time.sleep(0.2)
                 continue
Ejemplo n.º 49
0
        text = self.httpClient.geturlcon(url)
        result = {}
        try:
            result = self.parseHYDetailsMeta(text, id)
        except Exception, e:
            print traceback.format_exc()
            logger.info(e)
        if True:
            source = GateHuoYuanSource()
            try:
                source.save_from_a_source(result)
            except Exception, e:
                logger.error(e)
            else:
                logger.info("%s saved" % (id))
        else:
            logger.error(url + " is a null page")

    def getAndSaveCYFromId(self, id):
        url = "http://56gate.com/html/cy/cydetail_%s.html" % id
        try:
            GateCYSource.objects.get(webSiteId=id)
            logger.info("Zhuanxian %s already exsists" % id)
            return
        except:
            pass

        text = self.httpClient.geturlcon(url)
        result = {}
        try:
from third_party.reflection.reflection import *
from log.logger import logger
from serialize.serializer import Serializer
from config.arguments_parser import argparser
from serialize.abstract_serializer import AbstractSerializer


class StatusLogStarter(Starter):
    def __init__(self, generator=None, test_framework=None):
        Starter.__init__(self)
        self.generator = generator
        self.test_framework = test_framework
        if test_framework is None:
            self.test_framework = tests.tests()

    def _test(self):
        pass
        # for test in self.test_framework:
        #     assert isinstance(test, AbstractTest)
        #     Helper.run_test(test, self.generator.connection, self.generator.container,
        #                               [sys.stdout, open('result/res.html', 'w')], True)


starter = StatusLogStarter()
starter.start()

s = Serializer().serializer(argparser['serializer'])
for d in DLogGenerator().generate(20):
    logger.assert_true(isinstance(s, AbstractSerializer))
    logger.info('Serialized some stuff: {}'.format(s.serialize(d)))
    print s.serialize(d)