Esempio n. 1
0
def dialup():
    context = SSLContext(PROTOCOL_SSLv3)
    def _ssl():
        sslsession = SMTP_SSL(context=context)
        ext_log(sslsession.connect(getconf('smtp_server'), getconf('smtp_port')), 'SSL connection')
        return sslsession
    def  _starttls():
        tlssession = SMTP()
        ext_log(tlssession.connect(getconf('smtp_server'), getconf('smtp_port')), 'startTLS connection')
        tlssession.ehlo()
        if tlssession.has_extn('STARTTLS'):
            ext_log(tlssession.starttls(context=context), 'startTLS')
            tlssession.ehlo()
            return tlssession
        else:
            logger.warning('plaintext connection')
            return tlssession

    try:
        if getconf('smtp_use_ssl'):
            session = _ssl()
        else:
            session = _starttls()

        ext_log(session.login(getconf('smtp_user'), getconf('smtp_password')), 'login')
        return session

    except (SMTPException, SSLError) as ex:
            logger.error('SMTP error: %s' %(ex))
Esempio n. 2
0
def is_file_valid(model_path, save_file=False):
    assert model_path.endswith('.npy'), logger.error(
        'Invalid file provided {}'.format(model_path))
    if not save_file:
        assert os.path.exists(model_path), logger.error(
            'file not found: {}'.format(model_path))
    logger.info('[LOAD/SAVE] checkpoint path is {}'.format(model_path))
Esempio n. 3
0
def send_mail(to, messagetext, subject=None, **args):

    cc = args.get('cc', [])
    bcc = args.get('bcc', [])
    recipients = list(chain(to, cc, bcc))
    sender = args.get('sender', getconf('email_sender'))
    if not sender: sender = getconf('email_sender')
    footer = args.get('footer', getconf('email_footer'))
    if not footer: footer = getconf('email_footer')
    subjecttag = args.get('subjecttag', getconf('email_defaulttag'))
    subjectdate = args.get('subjectdate', getconf('email_subject_date'))
    files = args.get('files', [])

    logger.info('~' * 23)
    logger.info('sending new mail using %s:\n%d recipients ~ %d cc, %d bcc, %d files' %(sender, len(recipients), len(cc), len(bcc), len(files)))

    message = make_header(to, sender, cc, subject, subjecttag, subjectdate)
    message.attach(make_mime_text(messagetext, footer))
    [message.attach(make_mime_file(f)) for f in files]

    session = dialup()

    if session is not None:
        try:
            session.sendmail(sender, recipients, message.as_string().encode('UTF-8'))
        except SMTPException as ex:
            logger.error('smtp error: %s' %(ex))
            return ex
        else:
            logger.info('mail sent')
            return True
        finally:
            ext_log(session.quit(), 'quit')

    logger.info('end mail')
def pybackup(**options):
    """数据库备份"""
    # 检查磁盘空间
    if not util.check.check_disk_free():
        logger.error('磁盘空间不足,请释放空间')
        sys.exit(0)
    # 解析配置文件并获取参数
    backup_setting = util.config.get_config(options)
    # 记录本次执行的参数
    logger.debug(backup_setting)
    # 创建一个记录数据库实例
    rdb = util.record.RecordDB(backup_setting['record-type'], backup_setting['record-db-sqlite'],
                               backup_setting['record-db-mysql'])
    backup_setting.update({'rdb': rdb})
    # 检测参数合法性
    check_error = util.check.check_option(backup_setting)
    if check_error:
        logger.error(check_error)
        sys.exit(1)
    # 执行备份或恢复流程
    if options['restore']:
        # 执行恢复流程
        if 'backup-package' in backup_setting:
            # 从指定备份包恢复
            restore_from_specified_packages(backup_setting, backup_setting['backup-package'])
        else:
            # 从数据库获取恢复信息
            restore_process(backup_setting, backup_setting['host'], backup_setting['port'],
                            backup_setting['recover-point'])
    else:
        # 执行备份流程
        backup_process(backup_setting)
Esempio n. 5
0
def upload_dir(local, remote):
    from util import logger
    try:
        transport = paramiko.Transport((host, port))
        transport.connect(username=username, password=password)
        sftp = paramiko.SFTPClient.from_transport(transport)
        for root, dirs, files in os.walk(local):
            for filespath in files:
                local_file = os.path.join(root, filespath)
                a = local_file.replace(local, '')
                remote_file = os.path.join(remote, a)
                try:
                    sftp.put(local_file, remote_file)
                except Exception as e:
                    sftp.mkdir(os.path.split(remote_file)[0])
                    sftp.put(local_file, remote_file)
                # print(local_file + ' uploaded!')
            for name in dirs:
                local_path = os.path.join(root, name)
                a = local_path.replace(local, '')
                remote_path = os.path.join(remote, a)
                try:
                    sftp.mkdir(remote_path)
                    # print("mkdir path %s" % remote_path)
                except Exception as e:
                    print(e)
        # print('upload finish!')
        sftp.close()
    except Exception as e:
        logger.error('error occurred when uploading dir %s to %s', local,
                     remote)
        print(e)
    def run(self):
        self._build_session()

        with self._session as sess:
            self._build_model()

            while True:
                next_task = self._task_queue.get(block=True)

                if next_task[0] == parallel_util.WORKER_RUNNING:

                    self._num_envs_required = int(next_task[1])

                    # collect rollouts
                    traj_episode = self._play()
                    self._task_queue.task_done()
                    for episode in traj_episode:
                        self._result_queue.put(episode)

                if next_task[0] == parallel_util.WORKER_RUNNING_MT:

                    self._num_envs_required = int(next_task[1])

                    # collect rollouts
                    traj_episode = self._play()
                    self._task_queue.task_done()
                    for episode in traj_episode:
                        self._result_queue.put({
                            **episode, "task name":
                            self.task_name
                        })

                elif next_task[0] == parallel_util.AGENT_SET_WEIGHTS:
                    # set parameters of the actor policy
                    self._set_weights(next_task[1])
                    time.sleep(0.001)  # yield the process
                    self._task_queue.task_done()

                elif next_task[0] == parallel_util.END_ROLLOUT_SIGNAL or \
                        next_task[0] == parallel_util.END_SIGNAL:
                    # kill all the thread
                    # logger.info("kill message for worker {}".format(self._actor_id))
                    logger.info("kill message for worker")
                    self._task_queue.task_done()
                    break

                elif next_task[0] == parallel_util.AGENT_SET_WEIGHTS_MULTI:
                    weights = next_task[1]
                    self._set_weights(weights[self.task_name])
                    time.sleep(0.001)  # yield the process
                    self._task_queue.task_done()

                elif next_task[0] == parallel_util.AGENT_RENDER:
                    pass
                    #self._num_envs_required = 1
                    #self._render(next_task[1]['it'], next_task[1]['save_loc'])
                    #self._task_queue.task_done()
                else:
                    logger.error('Invalid task type {}'.format(next_task[0]))
            return
Esempio n. 7
0
def install_it(pkg, system='debian'):
    """
    Install a package only if it isn't already installed.
    If it was impossible the installation of the package the function return False,
    True otherwise.

    Possible values of system are:
    *    debian
    *    archlinux

    """
    if system == 'debian':
        cmd = 'sudo apt-get --assume-yes install ' + pkg
    elif system == 'archlinux':
        cmd = 'sudo pacman -S ' + pkg
    info('Installing the package ' + pkg + ' ...')
    if is_pack_installed(pkg):
        return True

    try:
        (st, out) = subprocess.getstatusoutput(cmd)
    except OSError:
        error('Maybe apt is not installed in this system.')
        st = 32512  # This the error code of 'command not found'
    return st == 0
    def read_power(self, SerialNumber):
        # page 53, table A3
        if not SerialNumber in self.caches_time:
            self.caches_time[SerialNumber] = 0
            self.caches_val[SerialNumber] = 0
        if not time.time(
        ) - self.caches_time[SerialNumber] > init.SERIAL_CACHE_TTL:
            # logger.info("CACHE HINT!!")
            return self.caches_val[SerialNumber]
        if self.ser_lock:
            while 1:
                if self.ser_lock:
                    # logger.info("Request Jamd")
                    time.sleep(init.SER_LOCK_RECHECK_TIME)
                else:
                    break

        addr = self.ser2addr(SerialNumber)

        self.ser_lock = True
        rsp = self.chn.read_power(addr)
        self.ser_lock = False

        if not rsp is None:
            self.caches_val[SerialNumber] = rsp
            self.caches_time[SerialNumber] = time.time()
            return rsp
        else:
            logger.error("COM %s, Addr %s no resp!" %
                         (self.COM_PORT, SerialNumber))
            return None
Esempio n. 9
0
def send_stop_timelapse(key):
    try:
        task = celery_app.send_task('cam_task.stop_timelapse_task', args=[key])
        return task.get()
    except Exception as e:
        logger.error(traceback.format_exc())
        raise e
Esempio n. 10
0
def check_xtrabackup_completed(log_text):
    """根据xtrabackup的日志检查xtrabackup是否执行成功"""
    last_row = log_text[log_text.rfind('\n') + 1:]
    if last_row[-13:] != 'completed OK!':
        logger.error('-' * 15 + ' ' + ' xtrabackup(innobackupex)备份结果校验失败 ' +
                     '-' * 15)
        sys.exit(1)
Esempio n. 11
0
 def __init__(self, url, method="GET", parameters=None, cookie=None, headers={}):
     try:
         # 解析url
         if type(url) == bytes:
             self.__url = url.decode("utf-8")
         if type(url) == str:
             self.__url = url
         logger.debug(self.__url)
         scheme, rest = urllib.parse.splittype(self.__url)
         # 拆分域名和路径
         logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
         self.__host_absolutely, self.__path = urllib.parse.splithost(rest)
         host_list = self.__host_absolutely.split(":")
         if len(host_list) == 1:
             self.__host = host_list[0]
             self.__port = 80
         elif len(host_list) == 2:
             self.__host = host_list[0]
             self.__port = host_list[1]
         # 对所传参数进行处理
         self.__method = method
         self.__data = parameters
         self.__cookie = cookie
         if parameters != None:
             self.__parameters_urlencode_deal = urllib.parse.urlencode(parameters)
         else:
             self.__parameters_urlencode_deal = ""
         self.__jdata = simplejson.dumps(parameters, ensure_ascii=False)
         self.__headers = headers
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 12
0
def send_aspsms(to, messagetext, **args):

    originator = args.get('originator', getconf('aspsms_originator'))
    if not originator: originator = getconf('aspsms_originator')
    flashing = args.get('flashing', getconf('aspsms_flashing'))
    maxchars = args.get('maxchars', getconf('aspsms_maxchars'))

    logger.info('~' * 23)
    logger.info('sending new aspsms using %s:\n%d recipients ~ flashing: %s' %(originator, len(to), flashing))

    message = wrap(messagetext, maxchars)

    try:
        for recipient in to:
            for text in message:
                payload = make_xml(recipient, originator, text, flashing)
                response = response_xml(post_xml(payload))
                if not response['ErrorCode'] == '1':
                    raise Exception('aspsms error: %s' %(response['ErrorDescription']))
    except Exception as ex:
        logger.error('error: %s' %(ex))
        return ex
    else:
        logger.info('aspsms sent')
        return True

    logger.info('end aspsms')
Esempio n. 13
0
def parseMessage(msgData, authors):
    """Parse the message contained in msgData.

     Authors should be a dict to provide a correspondence between the IDs as present in the msgData
     and eventually preferred aliases. If a key is not present, the ID itself is used as alias for all
     successive messages.

    :return: A string representing the parsed messages, or None if the value for
    a specific key was not found
    """
    try:
        localTimestamp = time.localtime(msgData["timestamp"] / 1000)
        dateAndTime = time.strftime("%Y.%m.%d %H:%M:%S", localTimestamp)
        body = msgData["body"].replace("\n", " ")
        authorId = msgData["author"].split(":")[1]
        if authorId not in authors:
            logger.warning(
                "Missing value for author ID {}. Using directly the ID for all successive messages"
                .format(authorId))
            authors[authorId] = str(authorId)
        author = authors[authorId]
        message = str(dateAndTime) + " " + author + " " + body
        return message
    except KeyError:
        logger.error("Parsing message. KeyError")
        logger.error(msgData)
        return None
Esempio n. 14
0
    def send_command(self, cmd_type, **kwds):
        '''
        '''
        import json
        packet = {}
        packet['Cmd'] = cmd_type
        packet['Seq'] = self.seq
        for key in kwds.keys():
            packet[key] = kwds[key]
        data = json.dumps(packet) + "\n"
        if isinstance(data, unicode):
            data = data.encode('utf8')


#        while not self._lock.acquire(False): #加锁
#            logger.warn('%s 延时发送: %s' % (time.clock(), data))
#            time.sleep(0.1)
        time0 = time.time()
        self._lock.acquire()
        time1 = time.time()
        delta = time1 - time0
        if self._enable_log and delta >= 0.05:
            logger.info('send wait %s S' % delta)
        if self._enable_log: logger.debug('send: %s' % (data[:512]))
        try:
            result = self.send(data)
        except Exception, e:
            # 避免因异常导致死锁
            logger.error('send %r error: %s' % (data, e))
            result = None
Esempio n. 15
0
def direct_clip(stacking, band_name, clip_extents, tile_id, rename, workdir):
    """Clip datatypes which require no special processing."""
    logger.info('     Start processing for band: %s', band_name)
    mosaic_filename = os.path.join(workdir, tile_id,
                                   tile_id + '_' + rename + '.tif')

    if os.path.exists(mosaic_filename):
        logger.warning("Skip previously generated result %s", mosaic_filename)
        return mosaic_filename

    warp_cmd = ('gdalwarp -te {extents}'
                ' -co "compress=deflate" -co "zlevel=9"'
                ' -co "tiled=yes" -co "predictor=2"').format(
                    extents=clip_extents)

    for stack in reversed(stacking):
        scene_name = util.ffind(workdir, stack['LANDSAT_PRODUCT_ID'],
                                '*' + band_name + '.tif')
        warp_cmd += ' ' + scene_name
    warp_cmd += ' ' + mosaic_filename
    util.execute_cmd(warp_cmd)

    logger.info('    End processing for %s as %s ', band_name, mosaic_filename)
    if not os.path.exists(mosaic_filename):
        logger.error('Processing failed to generate desired output: %s',
                     mosaic_filename)
    return mosaic_filename
 def read_power_by_IP(self, ip):
     if ip in globals.METERS_IP_MAP:
         SN = globals.METERS_IP_MAP[ip][0]
         SER = globals.METERS_IP_MAP[ip][1]
         return globals.COM_SHARD[SER].read_power(SN)
     else:
         logger.error("IP %s record not found" % ip)
Esempio n. 17
0
 def saveUser(self, user):
     assert type(user) == User, "类型错误"
     sql = "insert into User(userName,password,permission,wechatId,wechatName,registerTime,phoneNumber,emailAddress) values(%s,%s,%s,%s,%s,%s,%s,%s)"
     value = [
         user.userName,
         user.getPassword(), user.permission, user.wechatId,
         user.wechatName, user.registerTime, user.phoneNumber,
         user.emailAddress
     ]
     cursor = self.db.cursor()
     try:
         result = cursor.execute(sql, value)
         self.db.commit()
         cursor.close()
         return result
     except Exception as e:
         self.db.rollback()
         cursor.close()
         if e.args[0] == 1062:
             logger.warning(e.args[1])
             return 0
         logger.error("Failed:" + str(sql), exc_info=True)
         return -1
     pass
     return result
def parseMessage(msgData, authors):
    """Parse the message contained in msgData.

     Authors should be a dict to provide a correspondence between the IDs as present in the msgData
     and eventually preferred aliases. If a key is not present, the ID itself is used as alias for all
     successive messages.

    :return: A string representing the parsed messages, or None if the value for
    a specific key was not found
    """
    try:
        localTimestamp = time.localtime(msgData["timestamp"]/1000)
        dateAndTime = time.strftime("%Y.%m.%d %H:%M:%S", localTimestamp)
        body = msgData["body"].replace("\n", " ")
        authorId = msgData["author"].split(":")[1]
        if authorId not in authors:
            logger.warning("Missing value for author ID {}. Using directly the ID for all successive messages".format(authorId))
            authors[authorId] = str(authorId)
        author = authors[authorId]
        message = str(dateAndTime) + " " + author + " " + body
        return message
    except KeyError:
        logger.error("Parsing message. KeyError")
        logger.error(msgData)
        return None
Esempio n. 19
0
def publish(item, board, helper, channel, web):
    item_type = helper.item_type(item)
    post_process(item, board, helper, web)

    while True:
        try:
            channel.basic_publish(exchange='chan',
                                  routing_key="%s.%s.%s" %
                                  (chan, item_type, board),
                                  body=json.dumps(item,
                                                  separators=(',', ':'),
                                                  ensure_ascii=False,
                                                  sort_keys=True))

            if MONITORING:
                distance = datetime.utcnow() - datetime.utcfromtimestamp(
                    helper.item_mtime(item))
                monitoring.log([{
                    "measurement": chan,
                    "time": str(datetime.utcnow()),
                    "tags": {
                        "board": board
                    },
                    "fields": {
                        "distance": distance.total_seconds()
                    }
                }])
            break
        except Exception as e:
            logger.debug(traceback.format_exc())
            logger.error(str(e))
            channel = connect()
Esempio n. 20
0
def callback():
    code = request.args.get('code')
    if not code:
        logger.error("缺少code参数")
        return jsonify(Response.error(400, "缺少code参数"))
    logger.info("【code】" + str(code))
    # 通过code请求到access_token
    token_url = urls.get_token_url(code)
    resp = Request.get(token_url, to_dict=False)
    print(resp)
    try:
        access_token = re.findall("access_token=(.*?)&expires_in", resp)[0]
        logger.info("【access_token】" + str(access_token))
    except IndexError:
        logger.error('获取access_token错误')
        return jsonify(Response.error(400, "获取access_token错误"))
    session['qq_access_token'] = access_token
    # 通过access_token得到openid
    openid_url = urls.get_openid_url(access_token)
    resp = Request.get(openid_url)
    print(resp)
    openid = resp.get('openid')
    logger.info("【openid】" + str(openid))
    session['openid'] = openid
    return redirect(url_for('get_user_info'))
Esempio n. 21
0
    def _parse_stream(self, bot):
        stream = JSONStream(self.flow_user_api_key)
        gen = stream.fetch([self.channel], active=True)
        for data in gen:
            process_message = type(data) == dict and (data['event'] == "message" or data['event'] == "comment")
            if process_message and ("user" in data and self.user != data["user"]):
                self.spoken = False
                bot_input = BotInput()
                if type(data['content']) is dict:
                    bot_input.message = data["content"]['text']
                elif "content" in data:
                    bot_input.message = data["content"]
                else:
                    break
                if "user" in data and int(data["user"]) > 0:
                    try:
                        bot_input.nick = self.get_user_by_id(data["user"])["nick"]
                        self.user_id = data["user"]
                        if random.random() < (self.chattiness / 100):
                            logger.log("Randomly sending message to %s" % bot_input.nick)
                            self.private_message(data["user"], random.choice(self.responses["private_messages"]))
                    except Exception as e:
                        logger.error(e)
                        self.say(bot.responses["stranger"])
                elif "external_name" in data:
                    bot_input.nick = data["external_name"]
                else:
                    bot_input.nick = "anonymous"
                bot_input.bot = bot
                self.user_nick = bot_input.nick

                marvin.process(bot_input, self)
Esempio n. 22
0
 def wrapper(*args, **kwargs):
     prog = ''
     comment = args[1]
     # 找到要执行的程序,剔除掉环境变量赋值
     for arg in args[0]:
         if '=' not in str(arg):
             prog = str(arg)
             break
     logger.info('检查程序 ' + prog + ' 是否存在')
     # 检查程序是否存在
     if not shutil.which(prog):
         logger.error(prog + ' : 程序不存在')
         # raise ProgramError("Cannot locate binary: " + binary)
         sys.exit(prog)
     # 执行命令
     returncode, out_value, err_value = func(*args, **kwargs)
     # 检查命令行命令是否执行成功
     if returncode is None:
         logger.info('-' * 15 + ' ' + comment + ' 程序执行中 ' + '-' * 15)
     elif returncode == 0:
         logger.info('-' * 15 + ' ' + comment + ' 完成 ' + '-' * 15)
     else:
         logger.error('-' * 15 + ' ' + comment + ' 执行失败 ' + '-' * 15)
         sys.exit(1)
     # 根据xtrabackup的日志检查备份是否成功
     if prog in ['innobackupex', 'xtrabackup']:
         check_xtrabackup_completed(err_value.strip())
         logger.info('-' * 15 + ' ' + ' xtrabackup(innobackupex)备份结果校验成功 ' +
                     '-' * 15)
     return returncode, out_value, err_value
Esempio n. 23
0
    def sendMessages(self, user, text):
        '''
        '''
        url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=" + self.__accessToken
        data = {
            "touser": user.wechatId,
            "msgtype": "text",
            "agentid": WECHAT_CONFIG['AgentId'],
            "text": {
                "content": text
            },
            "safe": 0
        }
        try:
            result = requests.post(url, json.dumps(data))
            result = json.loads(result.text)
            if result['errcode'] != 0:
                self.getAccessToken()
                url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=" + self.__accessToken
                result = requests.post(url, json.dumps(data))
                result = json.loads(result.text)
        except Exception as e:
            logger.error("Post wechat push failed:{}".format(e), exc_info=True)

        if result['errcode'] != 0 or result['invaliduser'] != '':
            logger.error("Post wechat push failed.\n result :" +
                         json.dumps(result) + "\ndata: " + json.dumps(data))
            return -1

        return 0
Esempio n. 24
0
def install_it(pkg, system='debian'):
    """
    Install a package only if it isn't already installed.
    If it was impossible the installation of the package the function return False,
    True otherwise.

    Possible values of system are:
    *    debian
    *    archlinux

    """
    if system=='debian':
        cmd = 'sudo apt-get --assume-yes install '+pkg
    elif system=='archlinux':
        cmd = 'sudo pacman -S '+pkg
    info('Installing the package '+pkg+' ...')
    if is_pack_installed(pkg):
        return True
    
    try:
        (st, out) =  subprocess.getstatusoutput(cmd)
    except OSError:
        error('Maybe apt is not installed in this system.')
        st = 32512 # This the error code of 'command not found'
    return st==0
Esempio n. 25
0
def start():
    try:
        proxies_thread = threading.Thread(target=async_do)
        proxies_thread.setName("task-submitter")
        proxies_thread.start()
    except Exception as e:
        error("提交任务异常:", repr(e))
        start()
Esempio n. 26
0
def remove_backup_dir(path):
    """删除备份的文件夹"""
    try:
        shutil.rmtree(path)
        logger.info('备份目录 ' + path + ' 已删除')
    except Exception as e:
        logger.error('删除备份的文件夹失败')
        sys.exit(1)
Esempio n. 27
0
 def __check_args_length(must: int,
                         argname: str,
                         soft: bool = False) -> bool:
     if len(passed_args) < must:
         if not soft:
             logger.fatal('MISSING ARGUMENT: [%s]' % argname)
             raise Exception('manual interruption')
         logger.error('MISSING ARGUMENT: [%s]' % argname)
Esempio n. 28
0
 def __init__(self):
     try:
         current_dir = os.getcwd()
         self.__ini_config_file = BASE_DIR + "/conf/" + "config.ini"
         self.__conf_config_file = BASE_DIR + "/conf/" + "config.conf"
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 29
0
def start():
    try:
        proxies_thread = threading.Thread(target=async_do)
        proxies_thread.setName("tag-loader")
        proxies_thread.start()
    except Exception as e:
        error("待办任务获取异常:", repr(e))
        start()
Esempio n. 30
0
 def run(self):
     try:
         self.go()
     except Exception as e:
         logger.error("大鱼上传失败 任务id:{} error=>{}".format(self.id, e))
         self.msg = "上传未知失败  请检查上传日志 error:{}".format(e)
     self.browser.close()
     return self.status, self.aid, self.vid, self.msg
Esempio n. 31
0
def req_url(url):
    times = 0

    while True:
        times = times + 1

        proxies_ip = get_proxies_ip(url)

        # 豆瓣请求添加代理限制,必须使用代理访问
        if is_douban(url) and proxies_ip is None:
            # 没有代理地址可以用,需要阻塞处理,防止本地ip被封
            WORKER_WAIT.wait()
            continue

        if is_xici(url):
            info("西刺代理访问重复,", times, '次,休眠10s再次重试')
            # 防止接口访问频次过快
            if times > 0:
                time.sleep(10)

        headers = {
            'User-Agent': random.choice(USER_AGENTS),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Connection': 'keep-alive',
            'Accept-Encoding': 'gzip, deflate',
        }

        proxies = {}

        if proxies_ip is not None:
            agreement, location = split_proxies(proxies_ip)
            proxies[agreement] = location

        try:
            html = requests.get(url, proxies=proxies, timeout=5, headers=headers).text

            dom = etree.HTML(html)

            if html.strip == "":
                remove_proxies_ip(url, proxies_ip)
                continue

            # 判断接口返回数据是否正常,错误的数据则更换代理重新拉取
            if result_has_error(url, dom, proxies_ip):
                error(url, ",访问异常,页面内容:", html)
                remove_proxies_ip(url, proxies_ip)
                continue

            else:
                # 通过校验说明代理可用,加入优先级高的列表中
                add_effective_proxies_ip(proxies_ip)
                return html

        except Exception as e:
            error(url, "请求异常,需要更换代理地址。", proxies_ip)  # , repr(e)
            # 移除错误的代理地址
            remove_proxies_ip(url, proxies_ip)
Esempio n. 32
0
def send_start_timelapse(header: str, run_every: float, expire_at: str,
                         data: dict):
    try:
        task = celery_app.send_task('cam_task.start_timelapse_task',
                                    args=[header, run_every, expire_at, data])
        return task.get()
    except Exception as e:
        logger.error(traceback.format_exc())
        raise e
Esempio n. 33
0
def remove_dir(dir):
    """删除目录"""
    try:
        shutil.rmtree(dir)
    except Exception as e:
        logger.error('删除目录出错')
        logger.error('错误信息: ' + e)
    else:
        logger.info('删除目录 ' + dir + ' 成功')
Esempio n. 34
0
def receive_result(task_id):
    try:
        res = celery_app.AsyncResult(task_id)
        data = res.get()
        res.forget()
        return res
    except Exception as e:
        logger.error(traceback.format_exc())
        raise e
Esempio n. 35
0
    def _init_driver(self):
        '''初始化测试桩
        '''
        if AndroidSpyClient.server_opened(self._port): 
            # 字段赋值
            logger.info('port %d opened' % self._port)
            self._process['name'] = self._process_name
            self._process['id'] = self._device.adb.get_pid(self._process_name)
            if self.hello() != None:
                return

        timeout = 20
        time0 = time.time()
        proc_exist = False
        while time.time() - time0 < timeout:
            if not proc_exist:
                pid = self._device.adb.get_pid(self._process_name)
                if pid > 0:
                    proc_exist = True
                    self._process['name'] = self._process_name
                    self._process['id'] = pid
                    break

            time.sleep(1)

        if not proc_exist:
            raise RuntimeError('进程:%s 在%d秒内没有出现' % (self._process_name, timeout))
        
        timeout = 30
        
        try:
            if self._device.is_art():
                # Android 5.0上发现注入容易导致进程退出
                self._wait_for_cpu_low(20, 10)
            
            time0 = time.time()
            while time.time() - time0 < timeout:
                ret = self._device.adb.run_shell_cmd('.%s/inject %s' % (AndroidDriver.qt4a_path, self._process_name), True, timeout=120, retry_count=1)
                logger.debug('inject result: %s' % ret)
                if 'Inject Success' in ret: break
                elif 'Operation not permitted' in ret:
                    # 可能是进程处于Trace状态
                    pid = self._device.adb.get_pid(self._process_name)
                    status = self._device.adb.get_process_status(pid)
                    tracer_pid = int(status['TracerPid'])
                    if tracer_pid > 0:
                        if int(status['PPid']) == tracer_pid:
                            # 使用TRACEME方式防注入
                            raise Exception('应用使用了防注入逻辑,注入失败')
                        logger.warn('TracerPid is %d' % tracer_pid)
                        self._device.adb.kill_process(tracer_pid)
                time.sleep(1)
                
        except RuntimeError, e:
            logger.error('%s\n%s' % (e, self._device.adb.run_shell_cmd('ps')))
            logger.info(self._device.adb.dump_stack(self._process_name))
            raise e
Esempio n. 36
0
def send_color(path, **kwargs):
    try:
        task = celery_app.send_task('cv_task.cv_color',
                                    args=[path],
                                    kwargs=kwargs)
        return task.id
    except Exception as e:
        logger.error(traceback.format_exc())
        raise e
def ERROR(code,appcode,s):
    l = "%s %s %s" % (code,appcode,s)
    for ke in known_errors.iter('Error'):
        if(ke.attrib['ErrorCode'] == code and
           ke.attrib['AppCode'] == appcode):
            logger.warning(l)
            logger.warning("  >> %s",ke.attrib['Description'])
            return
    logger.error(l)
 def init(self, port, baud):
     self.COM_PORT = port
     self.chn = Channel()
     self.chn.open(port, int(baud))
     logger.info("Open %s ,baudrate is %s" % (port, baud))
     if not self.chn.isOpen():
         logger.error("Serial port %s open fail!" % port)
         return False
     return True
Esempio n. 39
0
def calc_nodata_9999_lineage(stacking, band_name, clip_extents, tile_id,
                             rename, workdir):
    """Clip scenes which have data outside the lineage, apply -9999 fill."""
    logger.info('     Start processing for band: %s', band_name)

    mosaic_filename = os.path.join(workdir, tile_id,
                                   tile_id + '_' + rename + '.tif')

    if os.path.exists(mosaic_filename):
        logger.warning("Skip previously generated result %s", mosaic_filename)
        return mosaic_filename

    temp_clipped_names = list()
    temp_masked_names = list()
    for level, stack in reversed(list(enumerate(stacking, start=1))):
        scene_name = util.ffind(workdir, stack['LANDSAT_PRODUCT_ID'],
                                '*' + band_name + '.tif')

        temp_name1 = mosaic_filename.replace('.tif',
                                             '_temp%d' % level + '.tif')
        temp_warp_cmd = ('gdalwarp -te {extents}'
                         ' -dstnodata "-9999" -srcnodata "-9999" {0} {1}')
        util.execute_cmd(
            temp_warp_cmd.format(scene_name, temp_name1, extents=clip_extents))
        temp_clipped_names.append(temp_name1)

        lineg_name = util.ffind(workdir, tile_id, '*LINEAGEQA.tif')
        temp_name2 = mosaic_filename.replace('.tif',
                                             '_temp%dM' % level + '.tif')
        temp_calc_cmd = ('gdal_calc.py -A {0} -B {lineage} --outfile {1}'
                         ' --calc="(A*(B=={level}) + (-9999*(B!={level})))"'
                         ' --NoDataValue=-9999')
        util.execute_cmd(
            temp_calc_cmd.format(temp_name1,
                                 temp_name2,
                                 lineage=lineg_name,
                                 level=level))
        temp_masked_names.append(temp_name2)

    temp_name = mosaic_filename.replace('.tif', '_temp.tif')
    temp_warp_cmd = 'gdalwarp {} {}'.format(' '.join(temp_masked_names),
                                            temp_name)
    util.execute_cmd(temp_warp_cmd)
    util.remove(*temp_masked_names + temp_clipped_names)

    warp_cmd = (
        'gdalwarp -dstnodata "-9999" -srcnodata "-9999" -co "compress=deflate"'
        ' -co "zlevel=9" -co "tiled=yes" -co "predictor=2" {} {}')
    util.execute_cmd(warp_cmd.format(temp_name, mosaic_filename))
    util.remove(temp_name)

    logger.info('    End processing for %s as %s ', band_name, mosaic_filename)
    if not os.path.exists(mosaic_filename):
        logger.error('Processing failed to generate desired output: %s',
                     mosaic_filename)
    return mosaic_filename
Esempio n. 40
0
 def ini_data(self, label, key, configFile=None):
     try:
         config = configparser()
         if configFile is None:
             configFile = self.__ini_config_file
         config.read(configFile)
         return config.get(label, key)
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 41
0
    def run(self):
        client = ConnectFactory().getConnect("redis",self.config)
        for msg in self.consumer:
            kafkamsg = self._decodemsg(msg)
            try:
                logger.info("message handling(%s)" % kafkamsg)
                jsondata = json.loads(kafkamsg['rawdata'])
                ObjectFactory.fromjson(jsondata["message"]).execute(client)

            except:
                logger.error("message execute error(%s)" % jsondata)
Esempio n. 42
0
 def md5(cls, strVar):
     try:
         # md5加密并小写化
         md5_var = hashlib.md5()
         md5_var.update(strVar.encode("utf-8"))
         # md5_var.update(strVar)
         sign = md5_var.hexdigest().lower()
         return sign
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 43
0
def getLocationId(location_name,redis_server):
   keys = redis_server.hkeys('locations')
   location_id = None
   for key in keys:
      value = redis_server.hget('locations',key)
      if value == location_name:
         location_id = key
   if not location_id:
     print 'can not find corresponding location id'
     logger.error("cannot find the location id")
     sys.exit(-1)
   else:
     print location_id
Esempio n. 44
0
File: ara.py Progetto: starenka/ara
def repost_replies(account_name):
    bf = open('.blacklist_%s'%account_name,'a+')
    blacklist = bf.read().splitlines()
    bf.close()

    rp = open('.reposted_%s'%account_name,'a+')
    reposted = rp.read().splitlines()

    account = settings.ACCOUNTS.get(account_name)

    try:
        logging.info('[%s] Getting last mentions offset'%account_name)
        bot = TwitterBot(settings.CONSUMER_KEY,settings.CONSUMER_SECRET,
                         account['key'],account['secret'])
        mentions = []
        try:
            mentions = bot.api.mentions()
            logging.info('[%s] Got %d mentions'%(account_name,len(mentions)))
        except Exception,e:
            logging.error('[%s] Failed to get mentions. %s'%(account_name,e))

        for mess in reversed(mentions):
            try:
                author = mess.author.screen_name
                if str(author) in blacklist:
                    logging.debug('[%s] Author %s blacklisted. Skipping.'%(account_name,str(author)))
                    continue
                if str(mess.id) in reposted:
                    logging.debug('[%s] Message #%s already reposted. Skipping.'%(account_name,str(mess.id)))
                    continue

                message = mess.text.split(' ')
                if message[0] != '@%s'%account_name:
                    continue #not a "@reply"

                trigger = message[1]
                triggers = dict(account['triggers'])
                if trigger not in triggers:
                    logging.warning('[%s] Bad message format, sending DM to author'%account_name)
                    bot.dm(author,account['not_triggered'])
                else:
                    len_params = {'message':'','user':author}
                    mess_len = len(triggers[trigger]%len_params)
                    params = {'message':bot.trim_message(' '.join(message[2:]),mess_len),'user':author}
                    message = triggers[trigger]%params
                    logging.info('[%s] Tweeting message %s'%(account_name,message))
                    bot.tweet(message)
                rp.write('%s\n'%mess.id)
            except Exception,e:
                logging.error('%s'%e)
                continue
Esempio n. 45
0
 def request_with_cookies(self):
     try:
         cookiejar = http.CookieJar()
         cookiejar.set_cookie(self.__cookie)
         opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookiejar))
         if self.__data is None:
             request = urllib.request.Request(self.__url)
         else:
             request = urllib.request.Request(self.__url, self.__data)
         html = opener.open(request).read()
         return html
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 46
0
def oyjtDict(amount, orderDate):
    try:
        oyjt_data = dict()
        oyjt_data["orderID"] = "307900"
        oyjt_data["orderPoint"] = "3.104"
        oyjt_data["mallName"] = "欧亚卖场"
        oyjt_data["storeName"] = "欧亚卖场"
        oyjt_data["productName"] = "辉山小枕240ml"
        oyjt_data["quantity"] = "16"
        oyjt_data["unitPice"] = "1.94"
        oyjt_data["amount"] = amount
        oyjt_data["orderDate"] = orderDate
        return oyjt_data
    except Exception as e:
        logger.error(e)
        logger.exception(u"捕获到错误如下:")
Esempio n. 47
0
 def loadPlugin(self):
     '''
     @summary:
         Plugin Moduleをロードします
     @todo: 
         仕様を考える 選択したモジュールをディレクトリにコピーするのか、モジュールパスを追加するのか 前者がシンプル
     '''
     filename = filedialog.askopenfilename(filetypes = [('Python Script', ('.py', '.pyc')),],
                                           initialdir = dirname(__file__))
     if filename != "":
         dist_path = join(self.plugin_path, basename(filename))
         if exists(dist_path):
             # ここで上書き確認
             logger.error("Already exists %s" % dist_path)
             return False
         shutil.copyfile(filename, dist_path)
         self.refreshList()
Esempio n. 48
0
 def conf_data(self, key, configFile=None):
     try:
         if configFile is None:
             configFile = self.__conf_config_file
         config_file = open(configFile, 'r')
         result = config_file.readlines()
         result_deal_list = []
         key_value_list = []
         for i in result:
             result_deal_list.append(i.strip('\n'))
         for j in result_deal_list:
             key_value_list.append(j.split('='))
         result_dict = dict(key_value_list)
         return result_dict[key]
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 49
0
def get_district_code():
    try:
        with open(DC_PATH) as file:
            data = file.read()
            district_list = data.split('\n')
        for node in district_list:
            # print node
            if node[10:11] != ' ':
                state = node[10:].strip()
            if node[10:11] == ' ' and node[12:13] != ' ':
                city = node[12:].strip()
            if node[10:11] == ' ' and node[12:13] == ' ':
                district = node[14:].strip()
                code = node[0:6]
                code_list.append({"state": state, "city": city, "district": district, "code": code})
    except Exception as e:
        logger.error(e)
        logger.exception(u"捕获到错误如下:")
Esempio n. 50
0
def ytdycDict(cardno, datetime_str_register, salesamt, datetime_str_buy):
    try:
        ytdyc_data = dict()
        ytdyc_data["card_no"] = cardno
        ytdyc_data["vipcardno"] = ""
        ytdyc_data["jointdate"] = datetime_str_register
        ytdyc_data["openid"] = ""
        ytdyc_data["points_flag"] = "0"
        ytdyc_data["score_type"] = "1"
        ytdyc_data["score"] = "13"
        ytdyc_data["time"] = datetime_str_buy
        ytdyc_data["salesamt"] = salesamt
        ytdyc_data["storecode"] = "1FD151"
        ytdyc_data["storename"] = u"奢侈品眼镜"
        return ytdyc_data
    except Exception as e:
        logger.error(e)
        logger.exception(u"捕获到错误如下:")
Esempio n. 51
0
def verify_archive_upload(youtube_id, filename):
    c_retries_allowed = 5
    c_retries = 0

    while c_retries < c_retries_allowed:
        try:
            request = urllib2.Request("http://s3.us.archive.org/KA-converted-{0}/{1}".format(youtube_id, filename))

            request.get_method = lambda: "HEAD"
            response = urllib2.urlopen(request)

            return response.code == 200
        except urllib2.HTTPError, e:
            c_retries += 1

            if c_retries < c_retries_allowed:
                logger.error("Error during archive upload verification attempt %s, trying again" % c_retries)
            else:
                logger.error("Error during archive upload verification final attempt: %s" % e)

            time.sleep(10)
Esempio n. 52
0
    def request(self):
        try:
            conn = client.HTTPConnection(self.__host, self.__port)
            if self.__method == "GET":
                self.path = self.__path + self.__parameters_urlencode_deal
                conn.request(self.__method, self.__path)
            if self.__method == "POST":
                if self.__headers == {"Content-type": "application/json"}:
                    conn.request(self.__method, self.__path, self.__jdata, self.__headers)
                if self.__headers == {"Content-type": "application/x-www-form-urlencoded"}:
                    conn.request(self.__method, self.__path, self.__data, self.__headers)

            response = conn.getresponse()
            result_origin = response.read()
            try:
                result = result_origin.decode("gb2312").encode("utf8")
            except:
                result = result_origin
            return result
        except Exception as e:
            logger.error(e)
            logger.exception(u"捕获到错误如下:")
Esempio n. 53
0
    def _parse_stream(self, bot):
        stream = JSONStream(self.token, self.room_id)
        gen = stream.fetch()
        for data in gen:
            process_message = type(data) == dict and ("text" in data)
            if process_message and ("fromUser" in data):
                from_user = data["fromUser"]["username"]
                self.spoken = False
                bot_input = BotInput()
                bot_input.message = data["text"]
                try:
                    bot_input.nick = from_user
                    #bot_input.bot_speaking = from_user.lower().startswith(self.nick.lower())
                    if from_user.lower().startswith(self.nick.lower()):
                        continue
                    self.user_id = data["fromUser"]["id"]
                except Exception as e:
                    logger.error(e)
                    self.say(bot.responses["stranger"])
                bot_input.bot = bot
                self.user_nick = bot_input.nick

                marvin.process(bot_input, self)
Esempio n. 54
0
def gennerator():
    try:
        global code_list
        code_list = []
        if not code_list:
            get_district_code()
        id = code_list[random.randint(0, len(code_list))]['code']  # 地区项
        id = id + str(random.randint(1930, 2013))  # 年份项
        da = date.today() + timedelta(days=random.randint(1, 366))  # 月份和日期项
        id = id + da.strftime('%m%d')
        id = id + str(random.randint(100, 300))  # ,顺序号简单处理
        i = 0
        count = 0
        weight = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]  # 权重项
        check_code = {'0': '1', '1': '0', '2': 'X', '3': '9', '4': '8', '5': '7', '6': '6', '7': '5', '8': '5',
                      '9': '3', '10': '2'}  # 校验码映射
        for i in range(0, len(id)):
            count = count + int(id[i]) * weight[i]
            id = id + check_code[str(count % 11)]  # 算出校验码
            return id
    except Exception as e:
        logger.error(e)
        logger.exception(u"捕获到错误如下:")
Esempio n. 55
0
def updateswitch(switch_ip,dms_location,redis_server):
   logger.info('start to update switch(%s)' % switch_ip)

   headers = {'content-type': 'application/json'}
   switch = {}
   switch['managementIp'] = switch_ip
   keys = redis_server.hkeys('locations')
   location_id = None
   for key in keys:
      value = redis_server.hget('locations',key)
      if value == dms_location:
         location_id = key
   if not location_id:
     print 'can not find corresponding location id'
     logger.error("cannot find the location id")
     sys.exit(-1)
   switch['locationId'] = location_id
   switch_json = json.dumps(switch)
   logger.info('request:%s' % switch_json)
   r = requests.post('%s/switch' % dso_url,data=switch_json,headers=headers)
   if r.status_code == 201:
      print 'notify dso to bind location and switch successfully.'
   else:
      print 'fail to notify dso to bind location and switch:status_code(%s),content(%s)' % (r.status_code,r.content)
Esempio n. 56
0
    def send_command(self, command):
        if importError:
            return None
        ssh_newkey = 'Are you sure you want to continue connecting'
        login_choices = [ssh_newkey, 'Password:'******'password:'******'ssh -o ConnectTimeout=10 {user}@{host}'.format(user = self.username, host=self.host)
            ssh = pexpect.spawn(ssh_cmd)
            i = ssh.expect(login_choices, timeout=12)
            if i == 0:
                ssh.sendline('yes')
                # Try again :)
                i = ssh.expect(login_choices)
            if i == 1 or i == 2:
                ssh.sendline(self.password)
            elif i == 3:
                logger.error("[%s] I either got key problems or connection timeout." % self.host)
                return None
            ssh.expect('>', timeout=60)
            # Ready to send cmd
            ssh.sendline(command)
            ssh.expect('</rpc-reply>', timeout=600)   # expect end of the XML

            xml = ssh.before # take everything printed before last expect()
            ssh.sendline('exit')
        except pexpect.ExceptionPexpect as e:
            msg = e.message.splitlines()[0]
            logger.error('[{}] unable to send command - error: {}'.format(self.host, msg))
            return None

        xml += '</rpc-reply>' # Add the end element as pexpect steals it
        # Remove the first line in the output which is the command sent
        # to JunOS.
        
        #Remove everything before command
        xml = self._strip_before(xml,command)
        try:
            xmldoc = minidom.parseString(xml)
        except ExpatError:
            logger.error('Malformed XML input from %s.' % self.host)
            print xml
            return None
        return xmldoc
Esempio n. 57
0
 def get_host(self):
     try:
         return self.__host
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 58
0
 def get_parameters_urlencode_deal(self):
     try:
         return self.__parameters_urlencode_deal
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Esempio n. 59
0
 def get_path(self):
     try:
         return self.__path
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
    def scrapeConversation(self, merge, offset, timestampOffset, chunkSize, limit, isGroupConversation):
        """Retrieves conversation messages and stores them in a JSON file
        If merge is specified, the new messages will be merged with the previous version of the conversation, if present
        """

        if merge:
            if not os.path.exists(self._directory + "conversation.json"):
                logger.error("Conversation not present. Merge operation not possible")
                return
            with open(self._directory + "conversation.json") as conv:
                convMessages = json.load(conv)
                numMergedMsgs = 0

        if not os.path.exists(self._directory):
            os.makedirs(self._directory)

        logger.info("Starting scraping of conversation {}".format(self._convID))

        messages = []
        msgsData = ""
        timestamp = "" if timestampOffset == 0 else str(timestampOffset)
        while self.CONVERSATION_ENDMARK not in msgsData:
            requestChunkSize = chunkSize if limit <= 0 else min(chunkSize, limit-len(messages))
            reqData = self.generateRequestData(offset, timestamp, requestChunkSize, isGroupConversation)
            logger.info("Retrieving messages " + str(offset) + "-" + str(requestChunkSize+offset))
            msgsData = self.executeRequest(reqData)
            jsonData = json.loads(msgsData)

            if jsonData and ('payload' in jsonData) and jsonData['payload']:
                if ('actions' in jsonData['payload']) and jsonData['payload']['actions']:
                    actions = jsonData['payload']['actions']

                    #case when the last message already present in the conversation
                    #is older newer than the first one of the current retrieved chunk
                    if merge and convMessages[-1]["timestamp"] > actions[0]["timestamp"]:
                        for i, action in enumerate(actions):
                            if convMessages[-1]["timestamp"] == actions[i]["timestamp"]:
                                numMergedMsgs = len(actions[i+1:-1]) + len(messages)
                                messages = convMessages + actions[i+1:-1] + messages
                                break
                        break

                    #We retrieve one message two times, as the first one of the previous chunk
                    #and as the last one of the new one. So we here remove the duplicate,
                    #but only once we already retrieved at least one chunk
                    if len(messages) == 0:
                        messages = actions
                    else:
                        messages = actions[:-1] + messages

                    #update timestamp
                    timestamp = str(actions[0]["timestamp"])
                else:
                    if 'errorSummary' in jsonData:
                        logger.error("Response error: " + jsonData['errorSummary'])
                    else:
                        logger.error("Response error. No messages found")
                        logger.error(msgsData)
                    return
            else:
                logger.error("Response error. Empty data or payload")
                logger.error(msgsData)
                logger.info("Retrying in " + str(self.ERROR_WAIT) + " seconds")
                time.sleep(self.ERROR_WAIT)
                continue

            offset += chunkSize
            if limit!= 0 and len(messages) >= limit:
                break

            time.sleep(self.REQUEST_WAIT)

        if merge:
            logger.info("Successfully merged {} new messages".format(numMergedMsgs))
            logger.info("Conversation total message count = {}".format(len(messages)))
        else:
            logger.info("Conversation scraped successfully. {} messages retrieved".format(len(messages)))

        self.writeMessages(messages)