Esempio n. 1
0
	def getCurrentlyLiveStreamers(self, serverChannelString):
		"""
		Get a string with all the currently live streamers that the provided channel follows. If there's only a few streamers live, more expansive info is shown per streamer
		:param serverChannelString: The server name followed by the channel name, separated by a space
		:return: A tuple, first entry is a success boolean, second one is the result string with either the error message or the currently live streamers
		"""
		streamerIdsToCheck = {}
		for streamername, streamerdata in self.watchedStreamersData.iteritems():
			if serverChannelString in streamerdata['followChannels'] or serverChannelString in streamerdata['reportChannels']:
				streamerIdsToCheck[streamerdata['clientId']] = streamername
		isSuccess, result = self.retrieveStreamDataForIds(streamerIdsToCheck)
		if not isSuccess:
			self.logError(u"[TwitchWatch] An error occurred during a manual live check. " + result)
			return (False, "I'm sorry, I wasn't able to retrieve data from Twitch. It's probably entirely their fault, not mine though. Try again in a little while")
		if len(result) == 0:
			return (True, "Nobody's live, it seems. Time for videogames and/or random streams, I guess!")
		#One or more streamers are live, show info on each of them
		reportStrings = []
		shouldUseShortReportString = len(result) >= 4  # Use shorter report strings if there's 4 or more people live
		for streamerId, streamerdata in result.iteritems():
			streamername = streamerIdsToCheck[streamerId]
			displayname = streamername
			if self.doesStreamerHaveNickname(streamername, serverChannelString):
				displayname = self.watchedStreamersData[streamername]['nicknames'][serverChannelString]
			url = u"https://twitch.tv/{}".format(streamername)
			if shouldUseShortReportString:
				reportStrings.append(u"{} ({})".format(displayname, url))
			else:
				reportStrings.append(StringUtil.removeNewlines(u"{}: {} [{}] ({})".format(IrcFormattingUtil.makeTextBold(displayname), streamerdata['title'], streamerdata['game_name'], url)))
		return (True, StringUtil.joinWithSeparator(reportStrings))
Esempio n. 2
0
def get_tdx_server_config():
    """"获取TDX服务器配置信息"""

    tdx_stock_server_config = {
        CONFIG.TDX_BEST_STOCK_SERVER: {},
        CONFIG.TDX_STOCK_SERVER_LIST: []
    }
    config = configparser.ConfigParser()
    # 判断是否存在配置文件
    if os.path.exists(CONFIG_PATH):
        try:
            config.read(CONFIG_PATH)
            try:
                # 获取服务器配置
                server_list = config.get(CONFIG.TDX_STOCK_SERVER,
                                         CONFIG.TDX_STOCK_SERVER_LIST)
                if StringUtil.is_not_empty(server_list):
                    tdx_stock_server_config[
                        CONFIG.TDX_STOCK_SERVER_LIST] = eval(server_list)
            finally:
                pass

            try:
                # 获取默认服务器
                best_stock_server = config.get(CONFIG.TDX_STOCK_SERVER,
                                               CONFIG.TDX_BEST_STOCK_SERVER)
                if StringUtil.is_not_empty(best_stock_server):
                    tdx_stock_server_config[
                        CONFIG.TDX_BEST_STOCK_SERVER] = eval(best_stock_server)
            finally:
                pass
        finally:
            pass

    return tdx_stock_server_config
Esempio n. 3
0
    def GET(self):
        now_time = datetime.now()
        week_ago = now_time - timedelta(7)
        weibo_lists = SinaDao.getweibo()
        time = str(week_ago).split(' ')[0].split('-')
        new_weibo_lists = []
        for weibo_list in weibo_lists:
            weibo_time = weibo_list['create_time'].split(' ')

            #weibo_time[5] is year weibo_time[1] is month weibo_time[2] is day
            if int(weibo_time[5]) == int(time[0]) and StringUtil.converttime(
                    weibo_time[1]) == int(
                        time[1]) and int(weibo_time[2]) >= int(time[2]):
                new_weibo_lists.append(weibo_list)
            elif int(weibo_time[5]) > int(time[0]):
                new_weibo_lists.append(weibo_list)
            elif int(weibo_time[5]) == int(time[0]) and StringUtil.converttime(
                    weibo_time[1]) > int(time[1]):
                new_weibo_lists.append(weibo_list)
        result = []
        result.append([len(new_weibo_lists)])
        result.append(jiebafenci(new_weibo_lists))
        result.append(WeiboUtil.trends_weekly())
        result = str(result).replace("u'", "'")

        return result
	def execute(self, fb, request):
		# global variables
		keyword = ''
		keyword_highlight = ''
		my_dst = 8
		pic_dict = {}

		'''
		logic of with friend search
		'''
		uid_commenter = request.get('friend_uid')
		if uid_commenter:
			pass
		else:
			return
		uid_owner = fb.uid
		timeUtil = TimeUtil()
		begin_time = timeUtil.timeBeforeNow(604800)

		fql = '''
		select post_id, fromid, time, text
		from comment
	       	where fromid = '%(uid_commenter)s'
		and post_id in (
			select concat('%(uid_owner)s', '_', status_id)
		       	from status
		       	where uid = '%(uid_owner)s'
			and time > %(begin_time)d
		)
		''' %{
			'uid_commenter': uid_commenter,
			'uid_owner': uid_owner,
			'begin_time': begin_time,
		}
		logging.info(fql)

		stringUtil = StringUtil()

		search_result = []
		result_set = fb.fql.query(fql)

		for result in result_set:
			feed_url = 'http://www.facebook.com/profile.php?id=%s&v=feed&story_fbid=%s' % (fb.uid, result['post_id'])
			message = stringUtil.replace_insensitive(result['text'], keyword, keyword_highlight)
			post_time = timeUtil.timezonetime(result['time'], timedelta(hours=my_dst))

			r = {}
			r['post_date'] = post_time
			r['avatar_url'] = ''
			r['message_url'] = feed_url
			r['message_text'] = message

			search_result.append(r)

		search_result.sort(cmp=compareByPostDateDesc)

		return search_result
Esempio n. 5
0
    def getCurrentlyLiveStreamers(self, serverChannelString):
        """
		Get a string with all the currently live streamers that the provided channel follows. If there's only a few streamers live, more expansive info is shown per streamer
		:param serverChannelString: The server name followed by the channel name, separated by a space
		:return: A tuple, first entry is a success boolean, second one is the result string with either the error message or the currently live streamers
		"""
        streamerIdsToCheck = {}
        for streamername, streamerdata in self.watchedStreamersData.iteritems(
        ):
            if serverChannelString in streamerdata[
                    'followChannels'] or serverChannelString in streamerdata[
                        'reportChannels']:
                streamerIdsToCheck[streamerdata['clientId']] = streamername
        isSuccess, result = self.retrieveStreamDataForIds(streamerIdsToCheck)
        if not isSuccess:
            self.logError(
                u"[TwitchWatch] An error occurred during a manual live check. "
                + result)
            return (
                False,
                "I'm sorry, I wasn't able to retrieve data from Twitch. It's probably entirely their fault, not mine though. Try again in a little while"
            )
        if len(result) == 0:
            return (
                True,
                "Nobody's live, it seems. Time for videogames and/or random streams, I guess!"
            )
        #One or more streamers are live, show info on each of them
        reportStrings = []
        shouldUseShortReportString = len(
            result
        ) >= 4  # Use shorter report strings if there's 4 or more people live
        for streamerId, streamerdata in result.iteritems():
            streamername = streamerIdsToCheck[streamerId]
            displayname = streamername
            if self.doesStreamerHaveNickname(streamername,
                                             serverChannelString):
                displayname = self.watchedStreamersData[streamername][
                    'nicknames'][serverChannelString]
            url = u"https://twitch.tv/{}".format(streamername)
            if shouldUseShortReportString:
                reportStrings.append(u"{} ({})".format(displayname, url))
            else:
                reportStrings.append(
                    StringUtil.removeNewlines(u"{}: {} [{}] ({})".format(
                        IrcFormattingUtil.makeTextBold(displayname),
                        streamerdata['title'], streamerdata['game_name'],
                        url)))
        return (True, StringUtil.joinWithSeparator(reportStrings))
Esempio n. 6
0
    def __init__(self):
        self.dao = CrawlDao.CrawlDao("modeldatas")
        self.fileDao = FileUtil.FileDao()
        self.stringUtil = StringUtil.StringUtil()
        self.sql_category_table = 'db_category'
        self.sql_part_table = 'db_category_part'

        # 已更新的分类 name+level+parent_id
        self.category_have_list = list()
        # 已更新的part name+level
        self.part_have_list = list()

        self.label_dict = {u'字标': '1', u'灯泡': '2', u'四滤': '3', u'': '0'}
        self.cat_kind_dict = {u'全车件': 1, u'易损件': 0}
        # 分类无法判断标识
        self.cat_kind_not = '3'
        self.aliss_name_text = ''
        self.label_text = ''

        # db_category level:dict(name+parent_id)
        self.level_name = self.init_category()
        # db_category_part name+cate_id:dict(part)
        # self.part_name = self.init_part()
        self.init_part()

        # name+level+parent_id:id
        self.category_id_dict = dict()

        # name+level+parent_id:vehicle_code
        self.category_code_dict = dict()

        # cat_id:cat_name
        self.cat_id_name = dict()
        # 是否是新增的类目
        self.is_new = False
Esempio n. 7
0
	def post(self):
		'''登陆请求
		paras:
			*	email_or_phone 		用户可以输入手机号或者邮箱来进行登陆
			*	pwd 				登陆密码
			*	remember_me			是否记住登陆状态(TODO)
		response:

		'''
		email_or_phone = self.get_argument('email_or_phone')
		pwd = self.get_argument('pwd')
		remember_me = self.get_argument('remember_me',default='off')

		enroll_type = StringUtil.is_email_or_phone(email_or_phone)
		if enroll_type == 'neither':
			#如果用户输入不是邮箱或者电话号,则登陆失败
			self.write(RESPONSE.WRONG_TYPE)
			return
		user = UserDao.get_user_by_email(email_or_phone) if enroll_type == 'email' else UserDao.get_user_by_phone(email_or_phone)
		if not user:
			#如果取不到用户,则登陆失败
			self.write(RESPONSE.WRONG_TYPE)
		elif hashlib.md5(pwd).hexdigest() != user['pwd']:
			#如果密码错误,则登陆失败
			self.write(RESPONSE.INVALID_PASSWORD)
		else:
			self.set_secure_cookie("tid",str(user['tid']))
			self.set_secure_cookie("timestamp", str(time.time()))
			self.write(RESPONSE.SUCCESS)
Esempio n. 8
0
	def post(self):
		'''用户注册
		paras:
			*	email_or_phone: 	用户可以输入自己的手机号或者邮箱用来注册
			*	user_name:			用来显示用户名称,不唯一
			*	pwd:				用户登录密码
			*	pwd_confirm:		用户登录密码确认
			*	accept:				用户是否接受使用条款
		'''
		email_or_phone = self.get_argument('email_or_phone')
		user_name = self.get_argument('user_name')
		pwd = self.get_argument('pwd')
		
		enroll_type = StringUtil.is_email_or_phone(email_or_phone)
		if enroll_type == 'neither':
			#如果用户输入不是邮箱或者电话号,则注册失败
			self.write(RESPONSE.WRONG_TYPE)
			return
		elif enroll_type == 'email':
			#如果email已经注册,则注册失败
			if UserDao.get_user_by_email(email_or_phone):
				self.write(RESPONSE.EMAIL_EXIST)
				return
		else:
			#如果手机号已经注册,则注册失败
			if UserDao.get_user_by_phone(email_or_phone):
				self.write(RESPONSE.PHONE_EXIST)
				return
		#进行用户注册流程
		UserDao.register(email_or_phone, enroll_type, user_name, pwd)
		self.write(RESPONSE.SUCCESS)
Esempio n. 9
0
    def __init__(self):
        self.dao = CrawlDao.CrawlDao()
        self.fileDao = FileUtil.FileDao()
        self.stringUtil = StringUtil.StringUtil()

        # p_id+code : id
        self.cate_id_dict = dict()
        # self.cate_name_dict = dict()
        cate_sql = 'SELECT cat_id,cat_name,cat_code,parent_id,vehicle_code,cat_level FROM db_category_dian WHERE cat_id > 2999 '
        cate_array = self.dao.db.get_data(cate_sql)
        for cate_data in cate_array:
            cat_id = str(cate_data['cat_id'])
            cat_code = str(cate_data['cat_code'])
            parent_id = str(cate_data['parent_id'])
            cat_level = str(cate_data['cat_level'])
            vehicle_code = str(cate_data['vehicle_code'])

            key = str(parent_id) + "_" + str(cat_code)
            if cat_level == '3':
                key += "_" + vehicle_code
            self.cate_id_dict[key] = cat_id
            # self.cate_name_dict[cat_id] = str(cate_data['cat_name'])

        # g_id:old_cat_id
        self.goods_dict = dict()
        goods_sql = "select cat_id,goods_id from db_goods where seller_id = 1"
        goods_array = self.dao.db.get_data(goods_sql)
        for goods_data in goods_array:
            self.goods_dict[str(goods_data['goods_id'])] = str(
                goods_data['cat_id'])
Esempio n. 10
0
	def execute(self, message):
		"""
		:type message: IrcMessage
		"""

		#First just get anything new, if there is any
		subprocess.check_output(['git', 'pull'])
		#Check if any new updates were pulled in
		outputLines = subprocess.check_output(['git', 'log', '--format=oneline']).splitlines()
		commitMessages = []
		linecount = 0
		for line in outputLines:
			lineparts = line.split(" ", 1)
			#If we've reached a commit we've already mentioned, stop the whole thing
			if lineparts[0] == self.lastCommitHash:
				break
			linecount += 1
			#Only show the last few commit messages, but keep counting lines regardless
			if len(commitMessages) < self.MAX_UPDATES_TO_DISPLAY :
				commitMessages.append(lineparts[1])
		if linecount == 0:
			replytext = u"No updates found, seems I'm up-to-date. I feel so hip!"
		elif linecount == 1:
			replytext = u"One new commit: {}".format(commitMessages[0])
		else:
			commitMessages.reverse()  #Otherwise the messages are ordered new to old
			replytext = u"{:,} new commits: {}".format(linecount, StringUtil.joinWithSeparator(commitMessages))
			if linecount > self.MAX_UPDATES_TO_DISPLAY:
				replytext += u"; {:,} older ones".format(linecount - self.MAX_UPDATES_TO_DISPLAY)
		#Set the last mentioned hash to the newest one
		self.lastCommitHash = outputLines[0].split(" ", 1)[0]

		message.reply(replytext, "say")
Esempio n. 11
0
    def getStreamerInfo(self, streamername, serverChannelString=None):
        """
		Get info on the provided streamer, if they're live
		:param streamername: The name of the streamer to get info on
		:param serverChannelString: The server-channel pair where the request originated from. Needed to determine whether we need to use a nickname
		:return: A tuple, first entry is a success boolean, second one is the result string with either the error message or the currently live streamers
		"""
        # Check if we happen to have the streamer's ID on file, saves retrieving it
        if streamername in self.watchedStreamersData:
            streamerId = self.watchedStreamersData[streamername]['clientId']
            displayName = self.getStreamerNickname(streamername,
                                                   serverChannelString)
        else:
            isSuccess, result = self.retrieveChannelInfo(streamername)
            if not isSuccess:
                return (False, result)
            streamerId = result['id']
            displayName = result['display_name']

        # Get stream info
        isSuccess, result = self.retrieveStreamDataForIds([streamerId], True)
        if not isSuccess:
            return (False, result)
        if len(result) == 0:
            return (
                True,
                u"{0} doesn't appear to be streaming at the moment. Maybe they've got some old streams you can watch though, here: https://twitch.tv/{0}/videos/all"
                .format(streamername))
        #Streamer is live, return info on them
        url = "https://twitch.tv/" + streamername
        return (True,
                StringUtil.removeNewlines(u"{}: {} [{}] ({})".format(
                    displayName, result[streamerId]['title'],
                    result[streamerId]['game_name'], url)))
Esempio n. 12
0
    def __init__(self):
        self.dao = CrawlDao.CrawlDao()
        self.fileDao = FileUtil.FileDao()
        self.stringUtil = StringUtil.StringUtil()

        self.table_mapping = 'db_category_mapping'

        self.vehicle_dict = {"商用车": "H", "乘用车": "C"}
        # 新分类的
        self.cate_dict = dict()
        cate_sql = "select cat_id,cat_name,vehicle_code,cat_level,parent_id from db_category_new "
        cate_array = self.dao.db.get_data(cate_sql)
        for cate_data in cate_array:
            cat_id = str(cate_data['cat_id'])
            cat_name = str(cate_data['cat_name'])
            cat_level = str(cate_data['cat_level'])
            vehicle_code = str(cate_data['vehicle_code'])
            parent_id = str(cate_data['parent_id'])
            key = cat_name + "_" + cat_level + "_" + parent_id

            if cat_level == '3':
                key += "_" + vehicle_code
            self.cate_dict[key] = cat_id

        # 老cate的图片数据
        self.old_cat_dict = dict()
        old_cat_sql = "select cat_name,category_thumb from db_category where cat_id < 2999 and parent_id = 0 and is_deleted = 'N'"
        old_cate_array = self.dao.db.get_data(old_cat_sql)
        for old_cate_data in old_cate_array:
            cat_name = str(old_cate_data['cat_name'])
            category_thumb = str(old_cate_data['category_thumb'])
            self.old_cat_dict[cat_name] = category_thumb
Esempio n. 13
0
 def _convertLineToProperFormat(self, rawDataLine, finalFile):
     try:
         foundGroups = re.search('"([A-Za-z\.\s\^/]+)"', rawDataLine)
         stockName = foundGroups.group(0)[1:-1].strip()
         if StringUtil.stringIsOnlyChars(stockName):
             finalFile.write(stockName + "\n")
     except AttributeError: logging.warn("Couldn't find a stock name in line: "+rawDataLine)
Esempio n. 14
0
	def getStreamerInfo(self, streamername, serverChannelString=None):
		"""
		Get info on the provided streamer, if they're live
		:param streamername: The name of the streamer to get info on
		:param serverChannelString: The server-channel pair where the request originated from. Needed to determine whether we need to use a nickname
		:return: A tuple, first entry is a success boolean, second one is the result string with either the error message or the currently live streamers
		"""
		# Check if we happen to have the streamer's ID on file, saves retrieving it
		if streamername in self.watchedStreamersData:
			streamerId = self.watchedStreamersData[streamername]['clientId']
			displayName = self.getStreamerNickname(streamername, serverChannelString)
		else:
			isSuccess, result = self.retrieveChannelInfo(streamername)
			if not isSuccess:
				return (False, result)
			streamerId = result['id']
			displayName = result['display_name']

		# Get stream info
		isSuccess, result = self.retrieveStreamDataForIds([streamerId], True)
		if not isSuccess:
			return (False, result)
		if len(result) == 0:
			return (True, u"{0} doesn't appear to be streaming at the moment. Maybe they've got some old streams you can watch though, here: https://twitch.tv/{0}/videos/all".format(streamername))
		#Streamer is live, return info on them
		url = "https://twitch.tv/" + streamername
		return (True, StringUtil.removeNewlines(u"{}: {} [{}] ({})".format(displayName, result[streamerId]['title'], result[streamerId]['game_name'], url)))
Esempio n. 15
0
    def printMessage(self, tasksDone):
        percentDone = float(tasksDone)/float(self._allTasksCount)*100.0
        now = time.clock()

        if now - self._lastTimePrinted > 1:
            if tasksDone > 0:
                timeRemaining = (now - self._start) / \
                            (float(tasksDone)/float(self._allTasksCount)) * \
                            (float(self._allTasksCount - tasksDone)/float(self._allTasksCount))
            else:
                timeRemaining = -1
            self._lastTimePrinted = now
            print self._message + ".... " + ("%.1f" % percentDone) + "% done, time ramaining",
            if timeRemaining < 0:
                print "unknown"
            else:
                print StringUtil.formatDuration(timeRemaining)
Esempio n. 16
0
    def GET(self):
        data = SinaDao.getmyinfo()
        if data['verified']:
            data['verified'] = u'是'
        else:
            data['verified'] = u'否'

        stime = data['created_at'].split(" ")

        data['created_at'] = str(stime[5]) + u"年" + str(
            StringUtil.converttime(stime[1])) + u"月" + str(
                stime[2]) + u"日" + StringUtil.convertweek(stime[0]) + stime[3]
        print data
        result = {}
        for i in data:
            result[str(i)] = data[i]
        return str(result).replace('\'', '\"')
Esempio n. 17
0
def get_sub_element_by_text(p_text, parent, translate_url, conversant_word_map, user=None):
    """
    进行 英文字符处理
    """
    result = []

    return_p_text = ''
    cur_u = None

    has_add_p_text = False

    for word in StringUtil.get_split_words(p_text):
            word_lower_case = word.lower()
            if_is_word = RegexUtil.is_word(word)

            if if_is_word:
                #添加到词频记录
                if not conversant_word_map.has_key(word_lower_case):
                    #没有在熟词列表内
                    result.append(return_p_text)
                    return_p_text = ''
                    has_add_p_text = True

                    cur_u = lxml.etree.SubElement(parent, "a")
                    jump_link = get_translate_word_url(word_lower_case, translate_url)
                    # cur_u.attrib['href'] = jump_link
                    # style="color:#DD4C53"

                    # cur_u.attrib['target'] = r"_blank"
                    # title_list = get_format_meaning(word_lower_case)

                    cur_u.attrib['class'] = 'recall_word'
                    cur_u.attrib['value'] = word_lower_case

                    # if not title_list:
                        # cur_u.attrib['style'] = r"color:#DD4CA0"
                        # cur_u.attrib['translate'] = u"未找到对应的翻译"
                    # else:
                        # cur_u.attrib['style'] = r"color:#FF0000"
                        # cur_u.attrib['translate'] = title_list

                    onclick = "return popitup2('%s')" %jump_link
                    # cur_u.attrib['onclick'] = onclick
                    cur_u.text = word
                    cur_u.tail = ''
                    result.append(cur_u)
                else:
                    return_p_text = return_p_text + word
                    has_add_p_text = False
            else:
                return_p_text = return_p_text + word
                has_add_p_text = False

    if not has_add_p_text:
        result.append(return_p_text)

    return result
Esempio n. 18
0
	def strip_and_clean(self, html):
		""" Cleans up the HTML structure and strips all tags. """
		html = "".join([line.strip() for line in html.split("\n")])
		html = re.sub('<!DOCTYPE.*?>', '', html)
		html = StringUtil.br2nl(html)

		# strip html
		html = ''.join(BeautifulSoup(html).findAll(text=True))

		return html
    def __init__(self):
        self.dao = CrawlDao.CrawlDao("athena_center", "local")
        self.fileDao = FileUtil.FileDao()
        self.stringUtil = StringUtil.StringUtil()

        self.insert_pic_data_list = list()
        self.update_goods_car_data_list = list()
        # mac pic
        self.max_pic_id = 0
        # {{pic_num,pic_index}:[oe,oe...]}
        self.pic_dict = dict()
Esempio n. 20
0
def __add_word_repeated_count(text, result):
    if text:
        word_list = StringUtil.get_split_words(text, word_only=True, lower=True)

        for word in word_list:
            if result.has_key(word):
                result[word] = result[word] + 1
            else:
                result[word] = 1

    return result
Esempio n. 21
0
def add(phone,owner,user):
	device = {
		'phone':phone,
		'active_code':StringUtil.active_code_generator(),
		'active':False,
		'owner':owner,
		'time':time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),
		'pull_info':[]
	}
	user['devices'].append(device)
	UserCollection.update({'tid':user['tid']},user)
	return RESPONSE.SUCCESS
Esempio n. 22
0
def get_log_config():
    """获取日志配置文件"""

    log_config = {
        CONFIG.LOG_PATH: DEF_LOG_PATH,
        CONFIG.LOG_LEVEL: logging.DEBUG
    }

    config = configparser.ConfigParser()
    # 判断是否存在配置文件
    if os.path.exists(CONFIG_PATH):
        try:
            config.read(CONFIG_PATH)
            try:
                # 获取日志目录配置
                log_path = config.get(CONFIG.LOG, CONFIG.LOG_PATH)
                if StringUtil.is_not_empty(log_path):
                    log_config[CONFIG.LOG_PATH] = log_path
            finally:
                pass

            try:
                # 获取日志等级
                log_level = config.get(CONFIG.LOG, CONFIG.LOG_LEVEL)
                if StringUtil.is_not_empty(
                        log_level) and log_level in LOG.TUPLE_LOG_LEVEL:
                    if str(log_level).upper() == LOG.ERROR:
                        log_config[CONFIG.LOG_LEVEL] = logging.ERROR
                    elif str(log_level).upper() == LOG.WARN:
                        log_config[CONFIG.LOG_LEVEL] = logging.WARN
                    elif str(log_level).upper() == LOG.INFO:
                        log_config[CONFIG.LOG_LEVEL] = logging.INFO
                    elif str(log_level).upper() == LOG.DEBUG:
                        log_config[CONFIG.LOG_LEVEL] = logging.DEBUG
            finally:
                pass
        finally:
            pass

    return log_config
Esempio n. 23
0
    def GET(self):
        now_time = datetime.now()
        week_ago = now_time - timedelta(7)
        weibo_lists = SinaDao.getweibo()
        time = str(week_ago).split(' ')[0].split('-')
        new_weibo_lists=[]
        for weibo_list in weibo_lists:
            weibo_time = weibo_list['create_time'].split(' ')

            #weibo_time[5] is year weibo_time[1] is month weibo_time[2] is day
            if  int(weibo_time[5])==int(time[0]) and StringUtil.converttime(weibo_time[1])==int(time[1]) and int(weibo_time[2])>=int(time[2]) :
                new_weibo_lists.append(weibo_list)
            elif int(weibo_time[5])>int(time[0]) :
                    new_weibo_lists.append(weibo_list)
            elif int(weibo_time[5])==int(time[0]) and StringUtil.converttime(weibo_time[1])>int(time[1]):
                new_weibo_lists.append(weibo_list)
        result=[]
        result.append([len(new_weibo_lists)])
        result.append(jiebafenci(new_weibo_lists))
        result.append(WeiboUtil.trends_weekly())
        result = str(result).replace("u'","'")
        
        return result
Esempio n. 24
0
	def calc_word_distance(self, str1, str2):
		""" Compares the two strings using Damerau-Levenshtein distance.
		"""
		# strip non alpha-numeric
		str1 = re.sub('[^a-z0-9]', lambda x: ' ', str1.lower())
		str2 = re.sub('[^a-z0-9]', lambda x: ' ', str2.lower())

		# cleanup spaces
		str1 = re.sub(' {2,}', lambda x: ' ', str1)
		str2 = re.sub(' {2,}', lambda x: ' ', str2)

		distance = 0
		if len(str2) > len(str1):
			distance = StringUtil.damerau_levenshtein(str1, str2[:len(str1)])

		# this extra check is to "weight" the values against the full string...
		# for instance, given str1="abc" and str2="abc but there is more", we want this to return a higher distance
		# than if given str1="abc but" and str2="abc but there is more"
		distance += StringUtil.damerau_levenshtein(str1, str2)

		# TODO might want to do an additional check on the exact initial strings, since the stripping we do would
		# cause "ab-cd" and "ab.cd" to match exactly the same, which would be bad
		return distance
Esempio n. 25
0
	def post(self):
		""" this handler supports http post """
		google_user = google_users.get_current_user()
		user = db.Query(Users).filter('user_id =', google_user.user_id()).get()
		if user is None:
			user = Users(user_id = google_user.user_id(), email = google_user.email())

		index = self.request.get('signature')
		user.signatures.pop(int(index))
		db.put(user)

		sigs = []
		for signature in user.signatures:
			sigs.append(StringUtil.nl2br(signature))

		self.response.out.write(json.dumps(sigs))
Esempio n. 26
0
    def GET(self):
        data = SinaDao.getmyinfo()
        if data['verified']:
            data['verified'] = u'是'
        else:
            data['verified'] = u'否'

        stime = data['created_at'].split(" ")
        

        data['created_at'] = str(stime[5])+u"年"+str(StringUtil.converttime(stime[1]))+u"月"+str(stime[2])+u"日"+StringUtil.convertweek(stime[0])+stime[3]
        print data
        result = {}
        for i in data:
            result[str(i)]=data[i]
        return str(result).replace('\'','\"')
Esempio n. 27
0
	def _process_html(self, html):
		'''
		 * 解析内容中的url,存入集合中
		'''
		urls = set()
		soup = BeautifulSoup(html)
		soup_res = soup.findAll(True, {'href': re.compile(self._target['grab_url_reg'])})
		for res in soup_res:
			url = res['href']
			#如果是资源文件,则丢弃
			if StringUtil.is_resource_url(url): continue
			urls.add(url)
		'''
		 * 将解析的url set存入到redis中
		'''
		for url in urls:
			self._redis.sset('sohu::url', url)
		print "     >>>>finish push %s urls in `sohu::url` of redis" %len(urls)
Esempio n. 28
0
    def _process_html(self, html):
        '''
		 * 解析内容中的url,存入集合中
		'''
        urls = set()
        soup = BeautifulSoup(html)
        soup_res = soup.findAll(
            True, {'href': re.compile(self._target['grab_url_reg'])})
        for res in soup_res:
            url = res['href']
            #如果是资源文件,则丢弃
            if StringUtil.is_resource_url(url): continue
            urls.add(url)
        '''
		 * 将解析的url set存入到redis中
		'''
        for url in urls:
            self._redis.sset('sohu::url', url)
        print "     >>>>finish push %s urls in `sohu::url` of redis" % len(
            urls)
Esempio n. 29
0
 def _get_html(self, url):
     #进行一些url的预处理,可扩展,目前没有处理
     url = StringUtil.process_url(url)
     headers = {
         #动态更改userAgent
         'User-Agent': AgentPoolUtil.getRandomUserAgent()
     }
     proxies = {
         #动态更改代理,https的tv.sohu.com用不上
         # 'http': AgentPoolUtil.getRandomProxy(),
     }
     try:
         response = requests.get(url, headers=headers, proxies=proxies)
     except Exception, e:
         print e
         print Exception
         print "请求错误拦截,因为url未必都靠谱,所以有可能会报错"
         print "%s Thread exit with error occurs"
         self._current_thread_count -= 1
         thread.exit_thread()
Esempio n. 30
0
	def _get_html(self, url):
		#进行一些url的预处理,可扩展,目前没有处理
		url = StringUtil.process_url(url)
		headers = {
			#动态更改userAgent
			'User-Agent': AgentPoolUtil.getRandomUserAgent()
		}
		proxies = {
			#动态更改代理,https的tv.sohu.com用不上
			# 'http': AgentPoolUtil.getRandomProxy(),
		}
		try:
			response  = requests.get(url, headers = headers, proxies=proxies)
		except Exception, e:
			print e
			print Exception
			print "请求错误拦截,因为url未必都靠谱,所以有可能会报错"
			print "%s Thread exit with error occurs"
			self._current_thread_count -= 1
			thread.exit_thread()
Esempio n. 31
0
	def get(self):
		""" this handler supports http get """
		data = {}
		if google_users.get_current_user():
			google_user = google_users.get_current_user()
			user = db.Query(Users).filter('user_id =', google_user.user_id()).get()

			if user is not None:
				data['token'] = user.pt_token
				data['havetoken'] = user.pt_token is not None

				data['has_signature'] = len(user.signatures) > 0
				data['has_email'] = user.pt_emails is not None and len(user.pt_emails) > 0

				if user.pt_emails is not None:
					data['emails'] = user.pt_emails

				if len(user.signatures) > 0:
					# convert \n to br so they display correctly
					sigs = []
					count = 0
					for signature in user.signatures:
						sig = {}
						sig['index'] = count
						sig['text'] = StringUtil.nl2br(signature)
						sig['is_html'] = re.search('<.*>', signature) is not None
						sigs.append(sig)
						count = count + 1

					data['signatures'] = sigs
				
				comment = db.Query(Comments).filter('user_id =', google_user.user_id()).get()
				data['has_comments'] = comment
				
			data['url'] = google_users.create_logout_url(self.request.uri)
			data['user'] = google_users.get_current_user()
		else:
			data['url'] = google_users.create_login_url(self.request.uri)

		path = os.path.join(os.path.dirname(__file__), 'index.html')
		self.response.out.write(template.render(path, data))
    def __init__(self):
        self.dao = CrawlDao.CrawlDao()
        self.stringUtil = StringUtil.StringUtil()
        self.sql_category_table = 'db_category'

        self.file_name = r'/Users/zxg/Desktop/old_cate.xlsx'

        self.cate_id_name = dict()
        self.cate_id_parent = dict()

        self.second_cat_list = list()
        # 其余行
        cate_sql = "select cat_id,cat_name,parent_id from db_category where is_deleted = 'N' order by cat_id"
        cate_array = self.dao.db.get_data(cate_sql)

        for cate_data in cate_array:
            cat_id = cate_data['cat_id']
            cat_name = cate_data['cat_name']
            parent_id = cate_data['parent_id']

            self.cate_id_name[cat_id] = cat_name
            self.cate_id_parent[cat_id] = parent_id
            if int(parent_id) != 0:
                self.second_cat_list.append(cat_id)
Esempio n. 33
0
import urllib
import urllib2
import cookielib
from dao import UserDao

#生成随机电话号码
def random_tel():
	tel_list = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
	tel_start = ['13','15','14','18']
	b = string.join(random.sample(tel_list,9)).replace(" ","")
	a = random.choice(tel_start)
	tel = a+b
	return tel

tel = random_tel()
email = StringUtil.token_generator() + '@qq.com'

#随机选取注册方式:手机|邮箱
def random_email_or_phone():
	random_dict = {
		'0':tel,
		'1':email
	}
	random_key = random.choice(random_dict.keys())
	random_value = random_dict[random_key]
	return random_value


def UniversalTest_post(postData,url):
	postData = urllib.urlencode(postData) 
	request = urllib2.Request(
Esempio n. 34
0
    def execute(self, message):
        """
		:type message: IrcMessage
		"""

        replytext = u""
        if 'openweathermap' not in GlobalStore.commandhandler.apikeys:
            replytext = u"No API key for OpenWeatherMap found, please tell my owner so they can fix this"
        elif message.messagePartsLength == 0:
            replytext = u"Please enter the name of a city"
        else:
            params = {
                "APPID": GlobalStore.commandhandler.apikeys['openweathermap'],
                "q": message.message,
                "units": "metric"
            }
            requestType = 'weather'
            if message.trigger == 'forecast':
                requestType = 'forecast/daily'
                params['cnt'] = 4  #Number of days to get forecast for
            try:
                req = requests.get("http://api.openweathermap.org/data/2.5/" +
                                   requestType,
                                   params=params,
                                   timeout=5.0)
                data = json.loads(req.text)
            except requests.exceptions.Timeout:
                replytext = u"Sorry, the weather API took too long to respond. Please try again in a little while"
            except ValueError:
                replytext = u"Sorry, I couldn't retrieve that data. Try again in a little while, maybe it'll work then"
                self.logError("[weather] JSON load error. Data received:")
                self.logError(req.text)
            else:
                if data['cod'] != 200 and data['cod'] != "200":
                    if data['cod'] == 404 or data['cod'] == '404':
                        replytext = u"I'm sorry, I don't know where that is"
                    else:
                        replytext = u"An error occurred, please tell my owner to look at the debug output, or try again in a little while ({}: {})".format(
                            data['cod'], data['message'])
                        self.logError("[weather] ERROR in API lookup:")
                        self.logError(data)
                else:
                    #We've got data! Parse it
                    def getWindDirection(angle):
                        #The highest wind angle where the direction applies
                        windDirectionTranslation = {
                            11.25: 'N',
                            33.75: 'NNE',
                            56.25: 'NE',
                            78.75: 'ENE',
                            101.25: 'E',
                            123.75: 'ESE',
                            146.25: 'SE',
                            168.75: 'SSE',
                            191.25: 'S',
                            213.75: 'SSW',
                            236.25: 'SW',
                            258.75: 'WSW',
                            281.25: 'W',
                            303.75: 'WNW',
                            326.25: 'NW',
                            348.75: 'NNW',
                            360.0: 'N'
                        }
                        windDirection = 'N'
                        for maxDegrees in sorted(
                                windDirectionTranslation.keys()):
                            if angle < maxDegrees:
                                break
                            else:
                                windDirection = windDirectionTranslation[
                                    maxDegrees]
                        return windDirection

                    def celsiusToFahrenheit(celsius):
                        return (celsius * 9 / 5) + 32

                    if message.trigger == 'weather':
                        dataAge = round((time.time() - data['dt']) / 60)
                        dataAgeDisplay = u""
                        if dataAge <= 0:
                            dataAgeDisplay = u"brand new"
                        else:
                            dataAgeDisplay = u"{dataAge:.0f} minute"
                            if dataAge > 1:
                                dataAgeDisplay += u"s"
                            dataAgeDisplay += u" old"
                            dataAgeDisplay = dataAgeDisplay.format(
                                dataAge=dataAge)

                        windString = u"{:.1f} m/s".format(
                            data['wind']['speed'])
                        #Only add a wind direction if we know it
                        if 'deg' in data['wind']:
                            windString += u", " + getWindDirection(
                                data['wind']['deg'])

                        #Not all replies include a placename or a countryname
                        placename = data['name'] if 'name' in data and len(
                            data['name']) > 0 else None
                        countryname = data['sys'][
                            'country'] if 'sys' in data and 'country' in data[
                                'sys'] and len(
                                    data['sys']['country']) > 0 else None
                        if placename and countryname:
                            replytext = u"{} ({})".format(
                                placename, countryname)
                        elif placename:
                            replytext = u"{}".format(placename)
                        elif countryname:
                            replytext = u"Somewhere in {}".format(countryname)
                        else:
                            replytext = u"Somewhere unknown"

                        #Add the actual weather info
                        replytext += u": {tempC:.1f}°C / {tempF:.1f}°F, {weatherType}. Wind: {windString}. Humidity of {humidity}% (Data is {dataAge})"
                        replytext = replytext.format(
                            tempC=data['main']['temp'],
                            tempF=celsiusToFahrenheit(data['main']['temp']),
                            weatherType=data['weather'][0]['description'],
                            windString=windString,
                            humidity=data['main']['humidity'],
                            dataAge=dataAgeDisplay)

                    else:
                        #Forecast
                        placename = data['city'][
                            'name'] if 'city' in data and 'name' in data[
                                'city'] and len(
                                    data['city']['name']) > 0 else None
                        countryname = data['city'][
                            'country'] if 'city' in data and 'country' in data[
                                'city'] and len(
                                    data['city']['country']) > 0 else None
                        replytext = u"Forecast for "
                        if placename and countryname:
                            replytext += u"{} ({})".format(
                                placename, countryname)
                        elif placename:
                            replytext += placename
                        elif countryname:
                            replytext += countryname
                        else:
                            replytext += u"somewhere unknown"
                        replytext += u": "

                        forecasts = []
                        for day in data['list']:
                            dayname = datetime.datetime.utcfromtimestamp(
                                day['dt']).strftime(u"%a").upper()

                            forecast = u"{dayname}: {minTempC:.0f}-{maxTempC:.0f}°C / {minTempF:.0f}-{maxTempF:.0f}°F, {weatherType}, {humidity}% hum., {windSpeed:.0f}m/s {windDir} wind"
                            forecast = forecast.format(
                                dayname=IrcFormattingUtil.makeTextBold(
                                    dayname),
                                minTempC=day['temp']['min'],
                                maxTempC=day['temp']['max'],
                                minTempF=celsiusToFahrenheit(
                                    day['temp']['min']),
                                maxTempF=celsiusToFahrenheit(
                                    day['temp']['max']),
                                humidity=day['humidity'],
                                windSpeed=day['speed'],
                                windDir=getWindDirection(day['deg']),
                                weatherType=day['weather'][0]['description'])
                            forecasts.append(forecast)
                        replytext += StringUtil.joinWithSeparator(forecasts)

        message.bot.sendMessage(message.source, replytext)
Esempio n. 35
0
 def printLastMessage(self):
     now = time.clock()
     print "Work done in " + StringUtil.formatDuration(now - self._start) + " seconds"
Esempio n. 36
0
        hostname = node[:node.find(":")]
        port = node[node.find(":") + 1:]
    server = {"ip": hostname,
              "port": port,
              "rest_username": options.username,
              "rest_password": options.password,
              "username": options.username,
              "password": options.password}
    print server
    v = None
    try:
        v = VBucketAwareMembaseClient(server, options.bucket)
        number_of_items = int(options.items)
        bar = ProgressBar(0, number_of_items, 77)
        old_bar_string = ""
        value = StringUtil.create_value("*", options.value_size)
        for i in range(0, number_of_items):
            key = "{0}-{1}".format(options.key_prefix, str(uuid.uuid4())[:5])
            if options.load_json:
                document = "\"name\":\"pymc-{0}\"".format(key, key)
                document = document + ",\"age\":{0}".format(random.randint(0, 1000))
                document = "{" + document + "}"
                a, b, c = v.set(key, 0, 0, document)
            else:
                a, b, c = v.set(key, 0, 0, value)
            a, b, c = v.get(key)

            bar.updateAmount(i)
            if old_bar_string != str(bar):
                sys.stdout.write(str(bar) + '\r')
                sys.stdout.flush()
Esempio n. 37
0
	def onProcess(self):
		self.list_request(self.request)

		#self.showFriendSelector()

		# global variables
		fb = self.fb
		keyword = ''
		keyword_highlight = ''
		my_dst = 8
		pic_dict = {}

		'''
		logic of with friend search
		'''
		uid_commenter = self.request.get('friend_uid')
		if uid_commenter:
			pass
		else:
			return
		uid_owner = fb.uid
		timeUtil = TimeUtil()
		begin_time = timeUtil.timeBeforeNow(604800)

		fql = '''
		select post_id, fromid, time, text
		from comment
	       	where fromid = '%(uid_commenter)s'
		and post_id in (
			select concat('%(uid_owner)s', '_', status_id)
		       	from status
		       	where uid = '%(uid_owner)s'
			and time > %(begin_time)d
		)
		''' %{
			'uid_commenter': uid_commenter,
			'uid_owner': uid_owner,
			'begin_time': begin_time,
		}
		logging.info(fql)

		stringUtil = StringUtil()

		search_result = []
		result_set = fb.fql.query(fql)

		self.out('result found %d' % len(result_set))
		for result in result_set:
			feed_url = 'http://www.facebook.com/profile.php?id=%s&v=feed&story_fbid=%s' % (fb.uid, result['post_id'])
			message = stringUtil.replace_insensitive(result['text'], keyword, keyword_highlight)
			post_time = timeUtil.timezonetime(result['time'], timedelta(hours=my_dst))

			r = {}
			r['post_date'] = post_time
			r['avatar_url'] = ''
			r['message_url'] = feed_url
			r['message_text'] = message

			search_result.append(r)

		search_result.sort(cmp=compareByPostDateDesc)

		avatar_url = "http://static.ak.fbcdn.net/pics/q_silhouette.gif"
		avatar_uid = uid_commenter
		if pic_dict.has_key(avatar_uid):
			avatar_url = pic_dict[avatar_uid]
		else:
			avatar_url = "http://static.ak.fbcdn.net/pics/q_silhouette.gif"

		withFriendView = StatusResultView()
		self.out(
			withFriendView.render_search_result('With-Friend', search_result, avatar_url)
			)
Esempio n. 38
0
	def execute(self, message):
		"""
		:type message: IrcMessage
		"""

		replytext = u""
		if 'openweathermap' not in GlobalStore.commandhandler.apikeys:
			replytext = u"No API key for OpenWeatherMap found, please tell my owner so they can fix this"
		elif message.messagePartsLength == 0:
			replytext = u"Please enter the name of a city"
		else:
			params = {"APPID": GlobalStore.commandhandler.apikeys['openweathermap'], "q": message.message, "units": "metric"}
			requestType = 'weather'
			if message.trigger == 'forecast':
				requestType = 'forecast/daily'
				params['cnt'] = 4  #Number of days to get forecast for
			try:
				req = requests.get("http://api.openweathermap.org/data/2.5/" + requestType, params=params, timeout=5.0)
				data = json.loads(req.text)
			except requests.exceptions.Timeout:
				replytext = u"Sorry, the weather API took too long to respond. Please try again in a little while"
			except ValueError:
				replytext = u"Sorry, I couldn't retrieve that data. Try again in a little while, maybe it'll work then"
				self.logError("[weather] JSON load error. Data received:")
				self.logError(req.text)
			else:
				if data['cod'] != 200 and data['cod'] != "200":
					if data['cod'] == 404 or data['cod'] == '404':
						replytext = u"I'm sorry, I don't know where that is"
					else:
						replytext = u"An error occurred, please tell my owner to look at the debug output, or try again in a little while ({}: {})".format(data['cod'], data['message'])
						self.logError("[weather] ERROR in API lookup:")
						self.logError(data)
				else:
					#We've got data! Parse it
					def getWindDirection(angle):
						#The highest wind angle where the direction applies
						windDirectionTranslation = {11.25: 'N', 33.75: 'NNE', 56.25: 'NE', 78.75: 'ENE', 101.25: 'E', 123.75: 'ESE',
													146.25: 'SE', 168.75: 'SSE', 191.25: 'S', 213.75: 'SSW', 236.25: 'SW',
													258.75: 'WSW', 281.25: 'W', 303.75: 'WNW', 326.25: 'NW', 348.75: 'NNW', 360.0: 'N'}
						windDirection = 'N'
						for maxDegrees in sorted(windDirectionTranslation.keys()):
							if angle < maxDegrees:
								break
							else:
								windDirection = windDirectionTranslation[maxDegrees]
						return windDirection

					def celsiusToFahrenheit(celsius):
						return (celsius * 9 / 5) + 32

					if message.trigger == 'weather':
						dataAge = round((time.time() - data['dt']) / 60)
						dataAgeDisplay = u""
						if dataAge <= 0:
							dataAgeDisplay = u"brand new"
						else:
							dataAgeDisplay = u"{dataAge:.0f} minute"
							if dataAge > 1:
								dataAgeDisplay += u"s"
							dataAgeDisplay += u" old"
							dataAgeDisplay = dataAgeDisplay.format(dataAge=dataAge)

						windString = u"{:.1f} m/s".format(data['wind']['speed'])
						#Only add a wind direction if we know it
						if 'deg' in data['wind']:
							windString += u", " + getWindDirection(data['wind']['deg'])

						#Not all replies include a placename or a countryname
						placename = data['name'] if 'name' in data and len(data['name']) > 0 else None
						countryname = data['sys']['country'] if 'sys' in data and 'country' in data['sys'] and len(data['sys']['country']) > 0 else None
						if placename and countryname:
							replytext = u"{} ({})".format(placename, countryname)
						elif placename:
							replytext = u"{}".format(placename)
						elif countryname:
							replytext = u"Somewhere in {}".format(countryname)
						else:
							replytext = u"Somewhere unknown"

						#Add the actual weather info
						replytext += u": {tempC:.1f}°C / {tempF:.1f}°F, {weatherType}. Wind: {windString}. Humidity of {humidity}% (Data is {dataAge})"
						replytext = replytext.format(tempC=data['main']['temp'], tempF=celsiusToFahrenheit(data['main']['temp']), weatherType=data['weather'][0]['description'],
													 windString=windString, humidity=data['main']['humidity'], dataAge=dataAgeDisplay)

					else:
						#Forecast
						placename = data['city']['name'] if 'city' in data and 'name' in data['city'] and len(data['city']['name']) > 0 else None
						countryname = data['city']['country'] if 'city' in data and 'country' in data['city'] and len(data['city']['country']) > 0 else None
						replytext = u"Forecast for "
						if placename and countryname:
							replytext += u"{} ({})".format(placename, countryname)
						elif placename:
							replytext += placename
						elif countryname:
							replytext += countryname
						else:
							replytext += u"somewhere unknown"
						replytext += u": "

						forecasts = []
						for day in data['list']:
							dayname = datetime.datetime.utcfromtimestamp(day['dt']).strftime(u"%a").upper()

							forecast = u"{dayname}: {minTempC:.0f}-{maxTempC:.0f}°C / {minTempF:.0f}-{maxTempF:.0f}°F, {weatherType}, {humidity}% hum., {windSpeed:.0f}m/s {windDir} wind"
							forecast = forecast.format(dayname=IrcFormattingUtil.makeTextBold(dayname), minTempC=day['temp']['min'], maxTempC=day['temp']['max'],
													   minTempF=celsiusToFahrenheit(day['temp']['min']), maxTempF=celsiusToFahrenheit(day['temp']['max']),
													   humidity=day['humidity'], windSpeed=day['speed'], windDir=getWindDirection(day['deg']), weatherType=day['weather'][0]['description'])
							forecasts.append(forecast)
						replytext += StringUtil.joinWithSeparator(forecasts)


		message.bot.sendMessage(message.source, replytext)
Esempio n. 39
0
##################初始化######################
# g = {}

# def setup_module(module):
# 	print "单元测试开始"
# 	g = {
# 		'email':123
# 	}

# def test_abc():
# 	print g['a']
# 	assert g['a']==1

##################UserDao##########################

email = StringUtil.token_generator() + "@qq.com"
user_name = 'xiaolin'
pwd = 'zxldmv'
enroll_type='email'
'''
用户注册流程
'''
def test_register():
	result = UserDao.register(email, enroll_type,user_name, pwd)
	assert result == 'success'

def test_get_user_by_email():
	user = UserDao.get_user_by_email(email)
	assert user['email'] == email

def test_get_user_by_tid():
Esempio n. 40
0
	def executeScheduledFunction(self):
		#Go through all our stored streamers, and see if we need to report online status somewhere
		#  If we do, check if they're actually online
		streamerIdsToCheck = {}  #Store as a clientId-to-streamername dict to facilitate reverse lookup in self.streamerdata later
		for streamername, data in self.watchedStreamersData.iteritems():
			if len(data['reportChannels']) > 0:
				#Ok, store that we need to check whether this stream is online or not
				# Because doing the check one time for all streamers at once is far more efficient
				streamerIdsToCheck[data['clientId']] = streamername

		if len(streamerIdsToCheck) == 0:
			#Nothing to do! Let's stop now
			return

		isSuccess, liveStreamDataById = self.retrieveStreamDataForIds(streamerIdsToCheck.keys())
		if not isSuccess:
			self.logError(u"[TwitchWatch] An error occurred during the scheduled live check. " + liveStreamDataById)
			#Still update the last checked time, so we do get results when the connection works again
			self.lastLiveCheckTime = time.time()
			return

		#If the last time we checked for updates was (far) longer ago than the time between update checks, we've probably been offline for a while
		# Any data we retrieve could be old, so don't report it, but just log who's streaming and who isn't
		if self.lastLiveCheckTime:
			shouldReport = time.time() - self.lastLiveCheckTime <= self.scheduledFunctionTime * 6
		else:
			shouldReport = True

		if not shouldReport:
			self.logDebug("[TwitchWatcher] Skipping reporting on live streams, since our last check was {} seconds ago, which is too long".format(time.time() - self.lastLiveCheckTime))

		self.lastLiveCheckTime = time.time()

		channelMessages = {}  #key is string with server-channel, separated by a space. Value is a list of tuples with data on streams that are live

		#Go through all the required IDs and check if the API returned info info on that stream. If so, store that data for display later
		for streamerId, streamername in streamerIdsToCheck.iteritems():
			#Check if the requested ID exists in the API reply. If it didn't, the stream is offline
			if streamerId not in liveStreamDataById:
				self.watchedStreamersData[streamername]['hasBeenReportedLive'] = False
			#If we have already reported the stream is live, skip over it now. Otherwise report that it has gone live
			elif not self.watchedStreamersData[streamername]['hasBeenReportedLive']:
				self.watchedStreamersData[streamername]['hasBeenReportedLive'] = True
				if shouldReport:
					#Stream is live, store some info to display later
					for serverChannelString in self.watchedStreamersData[streamername]['reportChannels']:
						#Add this stream's data to the channel's reporting output
						if serverChannelString not in channelMessages:
							channelMessages[serverChannelString] = []
						channelMessages[serverChannelString].append({'streamername': streamername, 'gameName': liveStreamDataById[streamerId]['game_name'],
																	 'title': liveStreamDataById[streamerId]['title']})

		#Save live status of all the streams
		self.saveWatchedStreamerData()

		if shouldReport:
			#And now report each online stream to each channel that wants it
			for serverChannelString, streamdatalist in channelMessages.iteritems():
				server, channel = serverChannelString.rsplit(" ", 1)
				#First check if we're even in the server and channel we need to report to
				if server not in GlobalStore.bothandler.bots or channel not in GlobalStore.bothandler.bots[server].channelsUserList:
					continue

				reportStrings = []
				#If we have a lot of live streamers to report, keep it short. Otherwise, we can be a bit more verbose
				useShortReportString = len(streamdatalist) >= 4
				for streamdata in streamdatalist:
					displayname = self.getStreamerNickname(streamdata['streamername'], serverChannelString)
					url = "https://twitch.tv/" + streamdata['streamername']
					#A lot of live streamers to report, keep it short. Just the streamer name and the URL
					if useShortReportString:
						reportStrings.append(u"{} ({})".format(displayname, url))
					# Only a few streamers live, we can be a bit more verbose
					else:
						reportStrings.append(StringUtil.removeNewlines(u"{}: {} [{}] ({})".format(IrcFormattingUtil.makeTextBold(displayname), streamdata['title'], streamdata['gameName'], url)))
				#Now make the bot say it
				GlobalStore.bothandler.bots[server].sendMessage(channel.encode("utf8"), u"Streamer{} went live: ".format(u's' if len(reportStrings) > 1 else u'') +
																StringUtil.joinWithSeparator(reportStrings), "say")
Esempio n. 41
0
	def execute(self, message):
		"""
		:type message: IrcMessage
		"""
		#Immediately check if there's any parameters, to prevent useless work
		if message.messagePartsLength == 0:
			message.reply("Please provide a term to search for. See '{}help {}' for an explanation how to use this command".format(message.bot.commandPrefix, message.trigger), "say")
			return

		searchType = message.messageParts[0].lower()

		addExtendedInfo = message.trigger == 'netrunner'

		#Check for update command before file existence, to prevent message that card file is missing after update, which doesn't make much sense
		if searchType == 'update' or searchType == 'forceupdate':
			if self.areCardfilesBeingUpdated:
				replytext = "I'm already updating!"
			elif not message.bot.isUserAdmin(message.user, message.userNickname, message.userAddress):
				replytext = "Sorry, only admins can use my update function"
			elif not searchType == 'forceupdate' and not self.shouldUpdate():
				replytext = "The last update check was done pretty recently, there's no need to check again so soon"
			else:
				replytext = self.updateCardFile()[1]
				#Since we're checking now, set the automatic check to start counting from now on
				self.resetScheduledFunctionGreenlet()
			message.reply(replytext, "say")
			return

		#Check if the data file even exists
		elif not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'data', 'NetrunnerCards.json')):
			if self.areCardfilesBeingUpdated:
				message.reply("I don't have my card database, but I'm solving that problem as we speak! Try again in, oh,  10, 15 seconds")
			else:
				message.reply("Sorry, I don't appear to have my card database. I'll try to retrieve it though! Give me 20 seconds, tops")
				gevent.spawn(self.updateCardFile)
				self.resetScheduledFunctionGreenlet()
			return

		#If we reached here, we're gonna search through the card store
		searchDict = {}
		# If there is an actual search (with colon key-value separator OR a random card is requested with specific search requirements
		if (searchType == 'search' and ':' in message.message) or (searchType == 'random' and message.messagePartsLength > 1):
			#Advanced search!
			if message.messagePartsLength <= 1:
				message.reply("Please provide an advanced search query too, in JSON format, so 'key1: value1, key2: value2'")
				return

			#Turn the search string (not the argument) into a usable dictionary, case-insensitive,
			searchDict = StringUtil.stringToDict(" ".join(message.messageParts[1:]).lower(), True)
			if len(searchDict) == 0:
				message.reply("That is not a valid search query. It should be entered like JSON, so 'name: Wall of Thorns, type: ICE,...'. ")
				return
		#If the searchtype is just 'random', don't set a 'name' field so we don't go through all the cards first
		#  Otherwise, set the whole message as the 'name' search, since that's the default search
		elif not searchType.startswith('random'):
			searchDict['title'] = message.message.lower()

		#Correct some values, to make searching easier (so a search for 'set' or 'sets' both work)
		searchTermsToCorrect = {'setname': ['set', 'sets'], 'flavor': ['flavour'], 'title': ['name']}
		for correctTerm, listOfWrongterms in searchTermsToCorrect.iteritems():
			for wrongTerm in listOfWrongterms:
				if wrongTerm in searchDict:
					if correctTerm not in searchDict:
						searchDict[correctTerm] = searchDict[wrongTerm]
					searchDict.pop(wrongTerm)

		#Turn the search strings into actual regexes
		regexDict = {}
		errors = []
		for attrib, query in searchDict.iteritems():
			try:
				#Since the query is a string, and the card data is unicode, convert the query to unicode before turning it into a regex
				regex = re.compile(unicode(query, encoding='utf8'), re.IGNORECASE)
			except (re.error, SyntaxError) as e:
				self.logError("[Netrunner] Regex error when trying to parse '{}': {}".format(query, e))
				errors.append(attrib)
			except UnicodeDecodeError as e:
				self.logError("[Netrunner] Unicode error in key '{}': {}".format(attrib, e))
				errors.append(attrib)
			else:
				regexDict[attrib] = regex
		#If there were errors parsing the regular expressions, don't continue, to prevent errors further down
		if len(errors) > 0:
			#If there was only one search element to begin with, there's no need to specify
			if len(searchDict) == 1:
				message.reply("An error occurred when trying to parse your search query. Please check if it is a valid regular expression, and that there are no non-UTF8 characters")
			#If there were more elements but only one error, specify
			elif len(errors) == 1:
				message.reply("An error occurred while trying to parse the query for the '{}' field. Please check if it is a valid regular expression without non-UTF8 characters".format(errors[0]))
			#Multiple errors, list them all
			else:
				message.reply("Errors occurred while parsing attributes: {}. Please check your search query for errors".format(", ".join(errors)))
			return

		#All entered data is valid, look through the stored cards
		with open(os.path.join(GlobalStore.scriptfolder, 'data', 'NetrunnerCards.json'), 'r') as jsonfile:
			cardstore = json.load(jsonfile)

		for index in xrange(0, len(cardstore)):
			carddata = cardstore.pop(0)

			#Then check if the rest of the attributes match
			for attrib in regexDict:
				if attrib not in carddata or not regexDict[attrib].search(carddata[attrib]):
					#If the wanted attribute is either not in the card, or it doesn't match, throw it out
					break
			#The else-block of a for-loop is executed when a for-loop isn't broken out of. So if everything matches, we get here
			else:
				cardstore.append(carddata)

		numberOfCardsFound = len(cardstore)
		#Pick a random card if needed and possible
		if searchType.startswith('random') and numberOfCardsFound > 0:
			cardstore = [random.choice(cardstore)]
			numberOfCardsFound = 1

		if numberOfCardsFound == 0:
			replytext = "Sorry, no card matching your query was found"
		elif numberOfCardsFound == 1:
			replytext = self.getFormattedCardInfo(cardstore[0], addExtendedInfo)
		else:
			nameMatchedCardFound = False
			replytext = ""
			#If there was a name search, check if the literal name is in the resulting cards
			if 'title' in searchDict:
				titleMatchIndex = None
				for index, card in enumerate(cardstore):
					if card['title'].lower() == searchDict['title']:
						titleMatchIndex = index
						break

				if titleMatchIndex:
					replytext = self.getFormattedCardInfo(cardstore[titleMatchIndex], addExtendedInfo)
					cardstore.pop(titleMatchIndex)
					numberOfCardsFound -= 1
					nameMatchedCardFound = True

			#Pick some cards to show
			maxCardsToList = 15
			if numberOfCardsFound > maxCardsToList:
				cardstore = random.sample(cardstore, maxCardsToList)
			cardnameText = ""
			for card in cardstore:
				cardnameText += card['title'].encode('utf-8') + "; "
			cardnameText = cardnameText[:-2]

			if nameMatchedCardFound:
				replytext += " ({:,} more match{} found: ".format(numberOfCardsFound, 'es' if numberOfCardsFound > 1 else '')
			else:
				replytext += "Your search returned {:,} cards: ".format(numberOfCardsFound)
			replytext += cardnameText
			if numberOfCardsFound > maxCardsToList:
				replytext += " and {:,} more".format(numberOfCardsFound - maxCardsToList)
			#Since the extra results list is bracketed when a literal match was also found, it needs a closing bracket
			if nameMatchedCardFound:
				replytext += ")"


		re.purge()  #Clear the stored regexes, since we don't need them anymore
		message.reply(replytext)
Esempio n. 42
0
    def executeScheduledFunction(self):
        #Go through all our stored streamers, and see if we need to report online status somewhere
        #  If we do, check if they're actually online
        streamerIdsToCheck = {
        }  #Store as a clientId-to-streamername dict to facilitate reverse lookup in self.streamerdata later
        for streamername, data in self.watchedStreamersData.iteritems():
            if len(data['reportChannels']) > 0:
                #Ok, store that we need to check whether this stream is online or not
                # Because doing the check one time for all streamers at once is far more efficient
                streamerIdsToCheck[data['clientId']] = streamername

        if len(streamerIdsToCheck) == 0:
            #Nothing to do! Let's stop now
            return

        isSuccess, liveStreamDataById = self.retrieveStreamDataForIds(
            streamerIdsToCheck.keys())
        if not isSuccess:
            self.logError(
                u"[TwitchWatch] An error occurred during the scheduled live check. "
                + liveStreamDataById)
            #Still update the last checked time, so we do get results when the connection works again
            self.lastLiveCheckTime = time.time()
            return

        #If the last time we checked for updates was (far) longer ago than the time between update checks, we've probably been offline for a while
        # Any data we retrieve could be old, so don't report it, but just log who's streaming and who isn't
        if self.lastLiveCheckTime:
            shouldReport = time.time(
            ) - self.lastLiveCheckTime <= self.scheduledFunctionTime * 6
        else:
            shouldReport = True

        if not shouldReport:
            self.logDebug(
                "[TwitchWatcher] Skipping reporting on live streams, since our last check was {} seconds ago, which is too long"
                .format(time.time() - self.lastLiveCheckTime))

        self.lastLiveCheckTime = time.time()

        channelMessages = {
        }  #key is string with server-channel, separated by a space. Value is a list of tuples with data on streams that are live

        #Go through all the required IDs and check if the API returned info info on that stream. If so, store that data for display later
        for streamerId, streamername in streamerIdsToCheck.iteritems():
            #Check if the requested ID exists in the API reply. If it didn't, the stream is offline
            if streamerId not in liveStreamDataById:
                self.watchedStreamersData[streamername][
                    'hasBeenReportedLive'] = False
            #If we have already reported the stream is live, skip over it now. Otherwise report that it has gone live
            elif not self.watchedStreamersData[streamername][
                    'hasBeenReportedLive']:
                self.watchedStreamersData[streamername][
                    'hasBeenReportedLive'] = True
                if shouldReport:
                    #Stream is live, store some info to display later
                    for serverChannelString in self.watchedStreamersData[
                            streamername]['reportChannels']:
                        #Add this stream's data to the channel's reporting output
                        if serverChannelString not in channelMessages:
                            channelMessages[serverChannelString] = []
                        channelMessages[serverChannelString].append({
                            'streamername':
                            streamername,
                            'gameName':
                            liveStreamDataById[streamerId]['game_name'],
                            'title':
                            liveStreamDataById[streamerId]['title']
                        })

        #Save live status of all the streams
        self.saveWatchedStreamerData()

        if shouldReport:
            #And now report each online stream to each channel that wants it
            for serverChannelString, streamdatalist in channelMessages.iteritems(
            ):
                server, channel = serverChannelString.rsplit(" ", 1)
                #First check if we're even in the server and channel we need to report to
                if server not in GlobalStore.bothandler.bots or channel not in GlobalStore.bothandler.bots[
                        server].channelsUserList:
                    continue

                reportStrings = []
                #If we have a lot of live streamers to report, keep it short. Otherwise, we can be a bit more verbose
                useShortReportString = len(streamdatalist) >= 4
                for streamdata in streamdatalist:
                    displayname = self.getStreamerNickname(
                        streamdata['streamername'], serverChannelString)
                    url = "https://twitch.tv/" + streamdata['streamername']
                    #A lot of live streamers to report, keep it short. Just the streamer name and the URL
                    if useShortReportString:
                        reportStrings.append(u"{} ({})".format(
                            displayname, url))
                    # Only a few streamers live, we can be a bit more verbose
                    else:
                        reportStrings.append(
                            StringUtil.removeNewlines(
                                u"{}: {} [{}] ({})".format(
                                    IrcFormattingUtil.makeTextBold(
                                        displayname), streamdata['title'],
                                    streamdata['gameName'], url)))
                #Now make the bot say it
                GlobalStore.bothandler.bots[server].sendMessage(
                    channel.encode("utf8"), u"Streamer{} went live: ".format(
                        u's' if len(reportStrings) > 1 else u'') +
                    StringUtil.joinWithSeparator(reportStrings), "say")
Esempio n. 43
0
    def execute(self, message):
        """
		:type message: IrcMessage
		"""
        url = "http://www.humblebundle.com/"
        #Allow for any special bundle search
        if message.messagePartsLength > 0:
            #Some bundles have a name with spaces in it. The URL replaces these with dashes, so we should too
            urlSuffix = message.message.replace(' ', '-').lower()
            if urlSuffix == 'store':
                message.reply(
                    "I'm sorry, I can't retrieve store data (yet (maybe))")
                return
            #Correct a possible typo, since the weekly bundle is called 'weekly' and not 'week'
            elif urlSuffix == 'week':
                url += 'weekly'
            else:
                url += urlSuffix

        #Only add all the games if the full trigger is used
        addGameList = message.trigger == 'humblebundle'

        try:
            pageDownload = requests.get(url, timeout=10.0)
        except requests.ConnectionError:
            message.reply(
                "Sorry, I couldn't connect to the Humble Bundle site. Try again in a little while!"
            )
            return
        except requests.exceptions.Timeout:
            message.reply(
                "Sorry, the Humble Bundle site took too long to respond. Try again in a bit!"
            )
            return

        if pageDownload.status_code != 200:
            self.logWarning(
                "[Humble] Page '{}' returned code {} instead of 200 (OK)".
                format(url, pageDownload.status_code))
            message.reply(
                "Sorry, I can't retrieve that bundle page. Either the site is down, or that bundle doesn't exist"
            )
            return

        page = BeautifulSoup(pageDownload.content, 'html.parser')

        #Get the part of the title up to the first opening parenthesis, since that's where the 'Pay what you wan't message starts
        title = page.title.string[:page.title.string.find('(') - 1]

        #First (try to) get a list of all the games with price requirements
        #Only if we actually need to show all the games
        if addGameList:
            lockedGames = {'BTA': [], 'Fixed': []}
            for lockImageElement in page.find_all('i', class_='hb-lock'):
                lockType = None
                if 'green' in lockImageElement.attrs['class']:
                    lockType = 'BTA'
                elif 'blue' in lockImageElement.attrs['class']:
                    lockType = 'Fixed'
                else:
                    continue
                #The game name is a sibling of the lock node, so parse the lock's parent text
                lockedGameElement = lockImageElement.parent
                #If the game name consists of a single line (and it's not empty) store that
                if lockedGameElement.string and len(
                        lockedGameElement.string) > 0:
                    lockedGames[lockType].append(
                        lockedGameElement.string.strip().lower())
                #Multiple lines. Add both the first line, and a combination of all the lines
                else:
                    lines = list(lockedGameElement.stripped_strings)
                    if len(lines) > 0:
                        lockedGames[lockType].append(lines[0].strip().lower())
                        #If there's multiple lines, join them and add the full title too
                        if len(lines) > 1:
                            lockedGames[lockType].append(
                                " ".join(lines).lower())

        #The names of the games (or books) are listed in italics in the description section, get them from there
        #Also do this if we don't need to list the games, since we do need a game count
        gamePriceCategories = {"PWYW": [], "BTA": [], "Fixed": []}
        gamecount = 0
        descriptionElement = page.find(class_='bundle-info-text')
        gameFound = False
        if not descriptionElement:
            self.logError("[Humble] No description element found!")
        else:
            descriptionGameList = []
            for paragraph in descriptionElement.find_all('p'):
                #If there is a bolded element, and it's at the start of the paragraph, AND we've already found names, we're done,
                #  because all the games are listed in the first paragraph
                boldedElement = paragraph.find('b')
                if boldedElement and paragraph.text.startswith(
                        boldedElement.text) and gameFound:
                    break
                #Otherwise, add all the titles listed to the collection
                for titleElement in paragraph.find_all(['i', 'em']):
                    gameFound = True
                    gamename = titleElement.text.strip(
                        " ,.;"
                    )  #Sometimes punctuation marks are included in the tag, remove those
                    #If the site lists two games after each other, they don't start a new HTML tag, so the game names
                    # get mushed together. Split that up
                    if "," in gamename:
                        gamenames = gamename.split(",")
                        for splitGamename in gamenames:
                            splitGamename = splitGamename.strip(" ,.;")
                            if len(splitGamename) > 0:
                                descriptionGameList.append(splitGamename)
                    #If there's no comma, it's just a single game name
                    else:
                        descriptionGameList.append(gamename)
            gamecount = len(descriptionGameList)

            #Now check to see which category the games we found belong to
            if addGameList:
                for gamename in descriptionGameList:
                    #See if this title is in the locked-games lists we found earlier
                    if gamename.lower() in lockedGames['BTA']:
                        gamePriceCategories['BTA'].append(gamename)
                    elif gamename.lower() in lockedGames['Fixed']:
                        gamePriceCategories['Fixed'].append(gamename)
                    else:
                        gamePriceCategories['PWYW'].append(gamename)

        #Totals aren't shown on the site immediately, but are edited into the page with Javascript. Get info from there
        totalMoney = -1.0
        contributors = -1
        avgPrice = -1.0
        timeLeft = u""
        for scriptElement in page.find_all('script'):
            script = scriptElement.text
            if script.count("'initial_stats_data':") > 0:
                #This script element contains data like the average price and the time left
                match = re.search("'initial_stats_data':(.+),", script)
                if not match:
                    self.logWarning(
                        "[Humble] Expected to find initial values, but failed:"
                    )
                    self.logWarning(script)
                else:
                    data = json.loads(match.group(1))
                    if 'rawtotal' in data:
                        totalMoney = data['rawtotal']
                    else:
                        self.logWarning(
                            "[Humble] Sales data found, but total amount is missing!"
                        )
                        self.logWarning(json.dumps(data))
                    if 'numberofcontributions' in data and 'total' in data[
                            'numberofcontributions']:
                        contributors = int(
                            data['numberofcontributions']['total'])
                    else:
                        self.logWarning("[Humble] Contributor data not found!")
                        self.logWarning(json.dumps(data))

                    if totalMoney > -1.0 and contributors > -1:
                        avgPrice = totalMoney / contributors
                    else:
                        self.logWarning(
                            "[Humble] Money raised and/or number of contributors not found!"
                        )
                        self.logWarning(json.dumps(data))

            #The time variable is in a different script than the other data, search for it separately
            timeLeftMatch = re.search(
                'var timing = \{"start": \d+, "end": (\d+)\};', script)
            if timeLeftMatch:
                timeLeft = DateTimeUtil.durationSecondsToText(
                    int(timeLeftMatch.group(1)) - time.time(), 'm')

            #If we found all the data we need, we can stop
            if avgPrice > -1.0 and timeLeft != u"":
                break

        if totalMoney == -1.0 or contributors == -1 or avgPrice == -1.0:
            replytext = u"Sorry, the data could not be retrieved. This is either because the site is down, or because of some weird bug. Please try again in a little while"
        else:
            replytext = u"{} has an average price of ${:.2f} and raised ${:,} from {:,} people.".format(
                title, round(avgPrice, 2), round(totalMoney, 2), contributors)
            if timeLeft != u"":
                replytext += u" It will end in {}.".format(timeLeft)
            replytext += u" It contains {:,} titles".format(gamecount)
            if addGameList:
                replytext += u":"
                #Add a list of all the games found
                for priceType in ('PWYW', 'BTA', 'Fixed'):
                    if len(gamePriceCategories[priceType]) > 0:
                        replytext += u" {}: {}".format(
                            IrcFormattingUtil.makeTextBold(priceType),
                            StringUtil.joinWithSeparator(
                                gamePriceCategories[priceType]))
                if not message.isPrivateMessage and len(
                        replytext) > Constants.MAX_MESSAGE_LENGTH:
                    replytext = replytext[:Constants.MAX_MESSAGE_LENGTH -
                                          5] + u"[...]"
                replytext += u" (itemlist may be wrong)"
            #Add the url too, so people can go see the bundle easily
            replytext += u" ({})".format(url)

        message.reply(replytext)
Esempio n. 44
0
 def test_is_empty(self):
     self.assertEquals(StringUtil.is_empty(""), True)
     self.assertEquals(StringUtil.is_empty(None), True)
     self.assertEquals(StringUtil.is_empty("test"), False)
Esempio n. 45
0
from selenium import webdriver
import time
from PIL import Image
from selenium.common.exceptions import UnexpectedAlertPresentException, NoAlertPresentException, NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from util import Util
import re
import util.StringUtil as StringUil
import os, sys
from util.BaiduApiUtil import BaiDuApi
from openpyxl import load_workbook  # pip install openpyxl

strUtil = StringUil.StringUtil()
file = "C://Users/62526/Desktop/Test/职称信息1.xlsx"
wb = load_workbook(file)
wb.guess_types = True
ws = wb.active
name_column = 2
id_num_column = 3
column4 = 4  # 在职单位名称
column5 = 5  # 学历
column6 = 6  # #注册证书信息
column7 = 7  # 注册类别
column8 = 8  # 注册号
column9 = 9  # 注册单位
column10 = 10  # 发证机关
column11 = 11  # 签发日期
column12 = 12  # 有效期
column13 = 13  # #职称证书信息
column14 = 14  # 证书名称
Esempio n. 46
0
	def execute(self, message):
		"""
		:type message: IrcMessage
		"""
		url = "http://www.humblebundle.com/"
		#Allow for any special bundle search
		if message.messagePartsLength > 0:
			#Some bundles have a name with spaces in it. The URL replaces these with dashes, so we should too
			urlSuffix = message.message.replace(' ', '-').lower()
			if urlSuffix == 'store':
				message.reply("I'm sorry, I can't retrieve store data (yet (maybe))")
				return
			#Correct a possible typo, since the weekly bundle is called 'weekly' and not 'week'
			elif urlSuffix == 'week':
				url += 'weekly'
			else:
				url += urlSuffix

		#Only add all the games if the full trigger is used
		addGameList = message.trigger == 'humblebundle'

		try:
			pageDownload = requests.get(url, timeout=10.0)
		except requests.ConnectionError:
			message.reply("Sorry, I couldn't connect to the Humble Bundle site. Try again in a little while!")
			return
		except requests.exceptions.Timeout:
			message.reply("Sorry, the Humble Bundle site took too long to respond. Try again in a bit!")
			return

		if pageDownload.status_code != 200:
			self.logWarning("[Humble] Page '{}' returned code {} instead of 200 (OK)".format(url, pageDownload.status_code))
			message.reply("Sorry, I can't retrieve that bundle page. Either the site is down, or that bundle doesn't exist")
			return

		page = BeautifulSoup(pageDownload.content, 'html.parser')

		#Get the part of the title up to the first opening parenthesis, since that's where the 'Pay what you wan't message starts
		title = page.title.string[:page.title.string.find('(') - 1]

		#First (try to) get a list of all the games with price requirements
		#Only if we actually need to show all the games
		if addGameList:
			lockedGames = {'BTA': [], 'Fixed': []}
			for lockImageElement in page.find_all('i', class_='hb-lock'):
				lockType = None
				if 'green' in lockImageElement.attrs['class']:
					lockType = 'BTA'
				elif 'blue' in lockImageElement.attrs['class']:
					lockType = 'Fixed'
				else:
					continue
				#The game name is a sibling of the lock node, so parse the lock's parent text
				lockedGameElement = lockImageElement.parent
				#If the game name consists of a single line (and it's not empty) store that
				if lockedGameElement.string and len(lockedGameElement.string) > 0:
					lockedGames[lockType].append(lockedGameElement.string.strip().lower())
				#Multiple lines. Add both the first line, and a combination of all the lines
				else:
					lines = list(lockedGameElement.stripped_strings)
					if len(lines) > 0:
						lockedGames[lockType].append(lines[0].strip().lower())
						#If there's multiple lines, join them and add the full title too
						if len(lines) > 1:
							lockedGames[lockType].append(" ".join(lines).lower())

		#The names of the games (or books) are listed in italics in the description section, get them from there
		#Also do this if we don't need to list the games, since we do need a game count
		gamePriceCategories = {"PWYW": [], "BTA": [], "Fixed": []}
		gamecount = 0
		descriptionElement = page.find(class_='bundle-info-text')
		gameFound = False
		if not descriptionElement:
			self.logError("[Humble] No description element found!")
		else:
			descriptionGameList = []
			for paragraph in descriptionElement.find_all('p'):
				#If there is a bolded element, and it's at the start of the paragraph, AND we've already found names, we're done,
				#  because all the games are listed in the first paragraph
				boldedElement = paragraph.find('b')
				if boldedElement and paragraph.text.startswith(boldedElement.text) and gameFound:
						break
				#Otherwise, add all the titles listed to the collection
				for titleElement in paragraph.find_all(['i', 'em']):
					gameFound = True
					gamename = titleElement.text.strip(" ,.;")  #Sometimes punctuation marks are included in the tag, remove those
					#If the site lists two games after each other, they don't start a new HTML tag, so the game names
					# get mushed together. Split that up
					if "," in gamename:
						gamenames = gamename.split(",")
						for splitGamename in gamenames:
							splitGamename = splitGamename.strip(" ,.;")
							if len(splitGamename) > 0:
								descriptionGameList.append(splitGamename)
					#If there's no comma, it's just a single game name
					else:
						descriptionGameList.append(gamename)
			gamecount = len(descriptionGameList)

			#Now check to see which category the games we found belong to
			if addGameList:
				for gamename in descriptionGameList:
					#See if this title is in the locked-games lists we found earlier
					if gamename.lower() in lockedGames['BTA']:
						gamePriceCategories['BTA'].append(gamename)
					elif gamename.lower() in lockedGames['Fixed']:
						gamePriceCategories['Fixed'].append(gamename)
					else:
						gamePriceCategories['PWYW'].append(gamename)

		#Totals aren't shown on the site immediately, but are edited into the page with Javascript. Get info from there
		totalMoney = -1.0
		contributors = -1
		avgPrice = -1.0
		timeLeft = u""
		for scriptElement in page.find_all('script'):
			script = scriptElement.text
			if script.count("'initial_stats_data':") > 0:
				#This script element contains data like the average price and the time left
				match = re.search("'initial_stats_data':(.+),", script)
				if not match:
					self.logWarning("[Humble] Expected to find initial values, but failed:")
					self.logWarning(script)
				else:
					data = json.loads(match.group(1))
					if 'rawtotal' in data:
						totalMoney = data['rawtotal']
					else:
						self.logWarning("[Humble] Sales data found, but total amount is missing!")
						self.logWarning(json.dumps(data))
					if 'numberofcontributions' in data and 'total' in data['numberofcontributions']:
						contributors = int(data['numberofcontributions']['total'])
					else:
						self.logWarning("[Humble] Contributor data not found!")
						self.logWarning(json.dumps(data))

					if totalMoney > -1.0 and contributors > -1:
						avgPrice = totalMoney / contributors
					else:
						self.logWarning("[Humble] Money raised and/or number of contributors not found!")
						self.logWarning(json.dumps(data))

			#The time variable is in a different script than the other data, search for it separately
			timeLeftMatch = re.search('var timing = \{"start": \d+, "end": (\d+)\};', script)
			if timeLeftMatch:
				timeLeft = DateTimeUtil.durationSecondsToText(int(timeLeftMatch.group(1)) - time.time(), 'm')

			#If we found all the data we need, we can stop
			if avgPrice > -1.0 and timeLeft != u"":
				break

		if totalMoney == -1.0 or contributors == -1 or avgPrice == -1.0:
			replytext = u"Sorry, the data could not be retrieved. This is either because the site is down, or because of some weird bug. Please try again in a little while"
		else:
			replytext = u"{} has an average price of ${:.2f} and raised ${:,} from {:,} people.".format(title, round(avgPrice, 2), round(totalMoney, 2), contributors)
			if timeLeft != u"":
				replytext += u" It will end in {}.".format(timeLeft)
			replytext += u" It contains {:,} titles".format(gamecount)
			if addGameList:
				replytext += u":"
				#Add a list of all the games found
				for priceType in ('PWYW', 'BTA', 'Fixed'):
					if len(gamePriceCategories[priceType]) > 0:
						replytext += u" {}: {}".format(IrcFormattingUtil.makeTextBold(priceType), StringUtil.joinWithSeparator(gamePriceCategories[priceType]))
				if not message.isPrivateMessage and len(replytext) > Constants.MAX_MESSAGE_LENGTH:
					replytext = replytext[:Constants.MAX_MESSAGE_LENGTH - 5] + u"[...]"
				replytext += u" (itemlist may be wrong)"
			#Add the url too, so people can go see the bundle easily
			replytext += u" ({})".format(url)

		message.reply(replytext)