def run_unfinish_content_is_null(): """ 重新获取内容空缺的文章 :return: """ sql = 'select classify, url from {_table} where length(content)=0 limit 10'.format(_table=mysql_table_sina_finance) df = pd.read_sql(sql, engine) for ix, row in df.iterrows(): content = _latest_content_by_beautifulsoup(row['url']) if content: sql = 'update {0} set content= "{1}" where classify="{2}" and url="{3}"'.format(mysql_table_sina_finance, content, row['classify'], row['url']) print sql engine.execute(sql)
def to_mysql(self): try: df = DataFrame( { "user_id": [self.user_id], "user_name": [self.user_name], "title": [self.title], "detail": [self.detail], "publish_time": [self.publish_time], "href": [self.href], "watch_count": [self.watch_count], "repost_count": [self.repost_count], "donate_count": [self.donate_count], "comment_count": [self.comment_count], #'is_top':[self.is_top], #'is_repost':[self.is_repost], #'repost_reason':[self.repost_reason "device": [self.device], # ] }, columns=[ "user_id", "user_name", "title", "detail", "publish_time", "href", "watch_count", "repost_count", "donate_count", "comment_count", "device", ], ) print df try: sql_del = "delete from {table} where user_id='{user_id}' and title='{title}' and publish_time='{publish_time}'".format( table=mysql_table_licaishi_viewpoint, user_id=self.user_id, title=self.title, publish_time=self.publish_time, ) engine.execute(sql_del) except Exception, e: print "delete error! ", str(e) df.to_sql(mysql_table_licaishi_viewpoint, engine, if_exists="append", index=False) return True
def to_mysql(self): try: df = DataFrame({'user_id':[self.user_id], 'user_name':[self.user_name], 'title':[self.title], 'detail': [self.detail], 'publish_time':[self.publish_time], 'capture_time':[self.capture_time], 'device':[self.device], 'href':[self.href], # 'repost_count':[self.repost_count], # 'donate_count':[self.donate_count], # 'comment_count':[self.comment_count], # 'is_top':[self.is_top], # 'is_repost':[self.is_repost], # 'repost_reason':[self.repost_reason] }, columns=['user_id', 'user_name', 'title', 'detail', 'publish_time', 'capture_time', 'device','href']) print df try: sql_del = "delete from {table} where user_id='{user_id}' and href='{href}' and publish_time='{publish_time}'".format( table = mysql_table_weixin_article, user_id = self.user_id, href = self.href, publish_time = self.publish_time ) engine.execute(sql_del) except Exception,e: print 'delete error!', str(e) df.to_sql(mysql_table_weixin_article, engine, if_exists='append', index=False) return True