Exemplo n.º 1
0
def sifting_date_str_stamp(date_str):
    """
    将获取到的日期转成有效的13位时间戳
    """
    try:
        if re.match(r'[0-9]+年\s\d月', "2016年 4月11日 10:30"):
            time_stamp = time.mktime(
                time.strptime("2016年 4月11日 10:30", '%Y年 %m月%d日 %H:%M'))
            return int(time_stamp * 1000)
        elif re.match(r'\d月', "4月11日 10:30"):
            year = format_time(time.localtime(), '%Y')
            date_str = year + "年 4月11日 10:30"
            time_stamp = time.mktime(
                time.strptime(date_str, '%Y年 %m月%d日 %H:%M'))
            return int(time_stamp * 1000)
        elif re.match(r'今天\d', "今天09:17"):
            date_str = format_time(time.localtime(), '%Y-%m-%d')
            date_str = date_str + " " + "今天09:17"[2:]
            time_stamp = time.mktime(time.strptime(date_str, '%Y-%m-%d %H:%M'))
            return int(time_stamp * 1000)
        elif re.match(r'\d+', "35分钟前"):
            minute = re.match(r'\d+', "35分钟前").group()
            that_time = datetime.datetime.now() + datetime.timedelta(
                minutes=-int(minute))
            return int(that_time.timestamp() * 1000)
    except Exception as err:
        logger.error("--> sifting real timestamp error, {} , {} ".format(
            date_str, str(err)))
    # 默认为当前日期
    return int(time.time() * 1000)
Exemplo n.º 2
0
def get_one(tagPo):
    try:
        mysql = MyPymysqlPool(ENV.MYSQL_CONF_TYPE)
        select_sql = "select id,`name`,href  from lp_tag "
        where_sql = "where "
        where_start_sql = ""
        if tagPo.id:
            where_start_sql += " and id=%d" % tagPo.id
        if tagPo.href:
            where_start_sql += " and href='%s'" % tagPo.href
        if where_start_sql and where_start_sql.__len__() > 0:
            where_start_sql = where_start_sql[4:]
            select_sql = select_sql + "where " + where_start_sql
        result = mysql.getOne(sql=select_sql)
        logger.info("--> select result " + str(result))
        if result:
            tagPo = TagPO()
            tagPo.__dict__.update(result)
            return tagPo
    except Exception as err:
        logger.error("--> select news error, sql :{}, err :{}".format(
            select_sql, str(err)))
    finally:
        mysql.dispose()
    return False
Exemplo n.º 3
0
def get_one(newsPo):
    try:
        mysql = MyPymysqlPool(ENV.MYSQL_CONF_TYPE)
        select_sql = "select id,module,title,synopsis,href,create_time,comment_count from lp_news "
        where_sql = "where "
        where_start_sql = ""
        if newsPo.id:
            where_start_sql += " and id=%d" % newsPo.id
        if newsPo.href:
            where_start_sql += " and href='%s'" % newsPo.href
        if where_start_sql and where_start_sql.__len__() > 0:
            where_start_sql = where_start_sql[4:]
            select_sql = select_sql + "where " + where_start_sql
        result = mysql.getOne(sql=select_sql)
        logger.info("--> select result " + str(result))
        if result:
            logger.info("id :{}, module:{}, title :{}, ".format(
                result['id'], result['module'], result['title']))
            newsObj = NewsPO()
            newsObj.__dict__.update(result)
            return newsObj
    except Exception as err:
        logger.error("--> select news error, sql :{}, err :{}".format(
            select_sql, str(err)))
    finally:
        mysql.dispose()
    return False
Exemplo n.º 4
0
def get_conn():
    # 打开数据库链接
    try:
        db = pymysql.connect(get_property('host'),
                             get_property('user'),
                             get_property('password'),
                             get_property('database'),
                             int(get_property('port')),
                             charset=ENV.MYSQL_CHARSET)
        return db
    except Exception as e:
        logger.error("--> get db failed, " + str(e))
        return None
Exemplo n.º 5
0
def insert(tagPo):
    mysql = MyPymysqlPool(ENV.MYSQL_CONF_TYPE)
    insert_sql = "insert into lp_tag(`name`, href) " \
                 "values('%s','%s')" % (str(tagPo.name), str(tagPo.href))
    try:
        result = mysql.insert(sql=insert_sql)
        tagPo.id = mysql.getInsertId()
        logger.info("--> insert news result : " + str(result))
        return result
    except Exception as err:
        logger.error(
            "--> insert news error, sql :{}, param :{}, \nerr :{}".format(
                insert_sql, class_to_dict(tagPo), str(err)))
    finally:
        mysql.dispose()
    return 0
Exemplo n.º 6
0
def insert(newsTagPo):
    mysql = MyPymysqlPool(ENV.MYSQL_CONF_TYPE)
    insert_sql = "insert into lp_news_tag( tag_id, news_id) " \
                 "values( %d, %d)" % (newsTagPo.tag_id, newsTagPo.news_id)
    try:
        result = mysql.insert(sql=insert_sql)
        newsTagPo.id = mysql.getInsertId()
        logger.info("--> insert lp_news_tag result : " + str(result))
        return result
    except Exception as err:
        logger.error(
            "--> insert lp_news_tag error, sql :{}, param :{}, \nerr :{}".
            format(insert_sql, class_to_dict(newsTagPo), str(err)))
    finally:
        mysql.dispose()
    return 0
Exemplo n.º 7
0
def insert(newsPo):
    mysql = MyPymysqlPool(ENV.MYSQL_CONF_TYPE)
    insert_sql = "insert into lp_news(module, title, synopsis, href, create_time, comment_count) " \
                 "values(%d,'%s','%s','%s',%d,'%s')" % (
                     newsPo.module, str(newsPo.title), str(newsPo.synopsis), str(newsPo.href), newsPo.create_time,
                     str(newsPo.comment_count))
    # param = (str(newsPo.module), str(newsPo.title), str(newsPo.synopsis), str(newsPo.href), str(newsPo.create_time),
    #          str(newsPo.comment_count))
    try:
        result = mysql.insert(sql=insert_sql)
        print("--> insert id :{}".format(mysql.getInsertId()))
        newsPo.id = mysql.getInsertId()
        logger.info("--> insert news result : " + str(result))
        return result
    except Exception as err:
        logger.error(
            "--> insert news error, sql :{}, param :{}, \nerr :{}".format(
                insert_sql, class_to_dict(newsPo), str(err)))
    finally:
        mysql.dispose()
    return 0
Exemplo n.º 8
0
def load_content_file_to_db(index):
    file = None
    try:
        file_path = global_consts[
            "sina_bs_folder"] + "20180313{}{}.txt".format(os.sep, index)
        file = open(file_path, 'r')
        ss = file.read()
        if ss:
            json_cont = json.loads(ss)
            if json_cont.__len__() > 1:
                if isinstance(json_cont, dict):
                    for k, v in json_cont.items():
                        news_po = NewsPO(module=10000,
                                         title=v['title'],
                                         synopsis=v['synopsis'],
                                         href=v['href'],
                                         create_time=sifting_date_str_stamp(
                                             v['create_time']),
                                         comment_count=sifting_comment_count(
                                             v['comment']))
                        logger.info(class_to_dict(news_po))
                        if news_po.href:
                            get_cont = news_mysql.get_one(
                                NewsPO(href=news_po.href))
                            if get_cont:
                                logger.warn(
                                    "--> news has exist, href :{}".format(
                                        get_cont.href))
                            else:
                                result = news_mysql.insert(news_po)
                                logger.info("--> insert news result : " +
                                            str(result))
                        else:
                            result = news_mysql.insert(news_po)
                            logger.info("--> insert news result : " +
                                        str(result))

                        tags = v['tags']
                        if tags and tags.__len__() > 0:
                            tag_objs = []
                            for tk in tags:
                                if tk:
                                    tag_obj = TagPO()
                                    for tv in tk:
                                        if str(tv).startswith('http'):
                                            tag_obj.href = tv
                                        else:
                                            tag_obj.name = tv
                                    tag_objs.append(tag_obj)
                            if tag_objs.__len__() > 0:
                                for toj in tag_objs:
                                    if tags_mysql.get_one(toj):
                                        logger.warn(
                                            "--> tag has exist : {}".format(
                                                toj.href))
                                    else:
                                        result = tags_mysql.insert(toj)
                                        logger.info(
                                            "--> insert tag result : {}".
                                            format(result))
                                        if toj.id and news_po.id:
                                            news_tag = NewsTag(
                                                tag_id=toj.id,
                                                news_id=news_po.id)
                                            result = news_tag_mysql.insert(
                                                news_tag)
                                            logger.info(
                                                "--> insert news_tag result :{}"
                                                .format(result))

    except Exception as err:
        logger.error("--> load content file error : " + str(err), err)
    finally:
        if file:
            file.close()