def get_cinfo_obejctid(conn_new, conn_old, terrace, objectid):
sql = 'select objectid,name,limiter,synonyms,exclude_limiter from object where objectid=%s' % (
objectid)
if terrace == 'new':
data = mysql.query_one(conn_new, sql)
else:
data = mysql.query_one(conn_old, sql)
return data
def get_site_id(conn,url):
domain = urlparse.urlparse(url).netloc
sql = "select siteid,name from xsite where url='%s';"%(domain)
data = mysql.query_one(conn, sql)
if not data:
isql = "insert into xsite(name,url) values('%s','%s');"%(domain,domain)
mysql.insert(conn, isql)
mysql.commit(conn)
data = mysql.query_one(conn, sql)
return data
def get_site_id(conn, url):
domain = urlparse.urlparse(url).netloc
sql = "select siteid,name from xsite where url='%s';" % (domain)
data = mysql.query_one(conn, sql)
if not data:
isql = "insert into xsite(name,url) values('%s','%s');" % (domain,
domain)
mysql.insert(conn, isql)
mysql.commit(conn)
data = mysql.query_one(conn, sql)
return data
def feed_data_to_xpost(conn,qualified_datas,terrace):
cursor = conn.cursor()
try:
n = len(qualified_datas)
key_counter_update_sql = 'UPDATE key_counter SET value=LAST_INSERT_ID(value+%d) WHERE name="xpostid";'%(n)
cursor.execute(key_counter_update_sql)
conn.commit()
key_counter_sql = 'SELECT LAST_INSERT_ID();'
if terrace=='old':
xpostid = mysql.query_one(conn, key_counter_sql)[0]
xpostid = xpostid - n
next_xpostid = MAX_INT_VALUE - xpostid - n
else:
xpostid = mysql.query_one(conn, key_counter_sql)[0]
next_xpostid = xpostid - n
#xpostid.write('%s %s\n'%(next_xpostid,next_xpostid+n))
except Exception,e:
print e
conn.rollback()
return -1
def feed_data_to_xpost(conn, qualified_datas, terrace):
cursor = conn.cursor()
try:
n = len(qualified_datas)
key_counter_update_sql = 'UPDATE key_counter SET value=LAST_INSERT_ID(value+%d) WHERE name="xpostid";' % (
n)
cursor.execute(key_counter_update_sql)
conn.commit()
key_counter_sql = 'SELECT LAST_INSERT_ID();'
if terrace == 'old':
xpostid = mysql.query_one(conn, key_counter_sql)[0]
xpostid = xpostid - n
next_xpostid = MAX_INT_VALUE - xpostid - n
else:
xpostid = mysql.query_one(conn, key_counter_sql)[0]
next_xpostid = xpostid - n
#xpostid.write('%s %s\n'%(next_xpostid,next_xpostid+n))
except Exception, e:
print e
conn.rollback()
return -1
next_xpostid = MAX_INT_VALUE - xpostid - n
else:
xpostid = mysql.query_one(conn, key_counter_sql)[0]
next_xpostid = xpostid - n
#xpostid.write('%s %s\n'%(next_xpostid,next_xpostid+n))
except Exception,e:
print e
conn.rollback()
return -1
insert_num = 0
for qualified_data in qualified_datas:
objectid = qualified_data['objectid']
date = qualified_data['pubtime'].strftime('%Y-%m-%d')
facet_sql = 'select id from xfacet where objectid=%s and type=1;'%(objectid)
try:
facetid = mysql.query_one(conn, facet_sql)[0]
except Exception,e:
#print e
continue
xentry_sql = 'select entryid from xentry where facetid=%s and date="%s";'%(facetid,date)
xentryid = mysql.query_one(conn, xentry_sql)
if not xentryid:
xentry_insert_sql = 'insert into xentry(facetid,date) values(%s,"%s");'%(facetid,date)
mysql.insert(conn, xentry_insert_sql)
mysql.commit(conn)
xentryid = mysql.query_one(conn, xentry_sql)
try:
xentryid = xentryid[0]
except Exception,e:
print e
return -1
else:
xpostid = mysql.query_one(conn, key_counter_sql)[0]
next_xpostid = xpostid - n
#xpostid.write('%s %s\n'%(next_xpostid,next_xpostid+n))
except Exception, e:
print e
conn.rollback()
return -1
insert_num = 0
for qualified_data in qualified_datas:
objectid = qualified_data['objectid']
date = qualified_data['pubtime'].strftime('%Y-%m-%d')
facet_sql = 'select id from xfacet where objectid=%s and type=1;' % (
objectid)
try:
facetid = mysql.query_one(conn, facet_sql)[0]
except Exception, e:
#print e
continue
xentry_sql = 'select entryid from xentry where facetid=%s and date="%s";' % (
facetid, date)
xentryid = mysql.query_one(conn, xentry_sql)
if not xentryid:
xentry_insert_sql = 'insert into xentry(facetid,date) values(%s,"%s");' % (
facetid, date)
mysql.insert(conn, xentry_insert_sql)
mysql.commit(conn)
xentryid = mysql.query_one(conn, xentry_sql)
try:
xentryid = xentryid[0]
except Exception, e:
def get_appnameid(appname):
sql = 'select id from appname_maps where e_appname="%s";'%(appname)
conn = mysql.connect('app_crawler', '192.168.241.17')
id = mysql.query_one(conn, sql)[0]
mysql.close(conn)
return id