def insert_url(self,table_name,url,types,scheme,body):
"""
数据入库
"""
if types == "GET":
result = db().query("select count(*) from {table_name} where scheme = '{scheme}'".format(table_name=table_name,scheme=scheme))
if result == (0L,):
print G+'[INFO] [URL] {url} [{types}] {W}'.format(types=types,url=url,W=W)
db().update("insert into {table_name}(url,types,scheme,time) values('{url}','{types}','{scheme}','{time_}')".format(table_name=table_name,url=url,types=types,scheme=scheme,time_=time.time()))
def open_spider(self,spider):
"""
初始化url数据表
"""
print '[*] Starting at {time_}'.format(time_=datetime.datetime.now().strftime('%H:%M:%S'))
db_name = db().db_name
table_name = spider.allowed_domains[0].replace('.','_')
count = db().query("select count(*) from information_schema.tables where table_schema = '{db_name}' and table_name = '{table_name}'".format(db_name=db_name,table_name=table_name))
if count == (0L,):
db().update("create table "+table_name+" (id int(10) NOT NULL primary key auto_increment,url varchar(400) NOT NULL,types char(4) NOT NULL,body varchar(1000),source varchar(20) not null default 'crawler',scheme varchar(100) not null,time varchar(30) not null)")
def add_context_to_db(body):
#
# add info to openstacktrack db
#
# get openstacktrack db config and open connection
db_openstacktrack = get_openstacktrack_conf()
db_track = mysql.db(db_openstacktrack['host'],db_openstacktrack['user'],db_openstacktrack['pass'],db_openstacktrack['db'])
# some data isn't present in every message
if '_context_request_id' in body:
context_request_id = body['_context_request_id']
else:
context_request_id = ''
if '_context_user_name' in body:
context_user_name = body['_context_user_name']
else:
context_user_name = ''
if '_context_remote_address' in body:
context_remote_address = body['_context_remote_address']
else:
context_remote_address = ''
event_type = body['event_type']
publisher_id = body['publisher_id']
timestamp =body['timestamp']
query = """INSERT INTO track
(context_request_id,context_user_name,context_remote_address,event_type,publisher_id,timestamp)
VALUES ('%s','%s','%s','%s','%s','%s')""" %
(context_request_id,context_user_name,context_remote_address,event_type,publisher_id,timestamp)
def compute_instance_delete_end(body):
# note for delete.end event, we don't get any IP info, just hostname
logger.debug("in compute_instance_delete_end()")
#
# Incorporate interfacing modules here
#
#
# ex. Remove host info from PowerDNS MySQL backend
import modules.powerdns.powerdns as powerdns, modules.powerdns.powerdns_config as powerdns_config
db_powerdns_config = powerdns_config.get_db_powerdns_config()
db_powerdns = mysql.db(db_powerdns_config['host'],db_powerdns_config['user'],db_powerdns_config['pass'],db_powerdns_config['db'])
logger.info("Removing host record from DNS database.")
remove_name = body['payload']['hostname']
#try:
powerdns.remove_record(remove_name,'domain_name','A')
#except
# blah
# returning true will signal to calling function to ACK rabbitmq message
return True
def _db(self):
if self.__db:
return self.__db
else:
return db()
#encoding:utf-8
import mysql
import datetime
import sys
default_encoding = 'utf-8'
if sys.getdefaultencoding() != default_encoding:
reload(sys)
sys.setdefaultencoding(default_encoding)
conn = mysql.db()
now = datetime.datetime.now()
now_second = datetime.datetime.now().strftime('%s')
def rate():
rate_sql = """
select category, value from zx_bonus_rule where category in ('rongzidun', 'jiangjinbi', 'lovemoney', 'platmoney', 'taxmoney')
"""
rates = conn.query(rate_sql)
conn.close()
if rates:
rates = rates
else:
rates = ({
'category': 'rongzidun',
'value': 25
}, {
'category': 'jiangjinbi',
#!/usr/bin/python
#
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
#
import time
import logging
import mysql
logger = logging.getLogger('root')
# get openstacktrack config and open connection
db_powerdns = get_db_powerdns_config()
db_dns = mysql.db(db_powerdns['host'],db_powerdns['user'],db_powerdns['pass'],db_powerdns['db'])
powerdns_config = get_powerdns_config()
def remove_record(name,zone,type):
#query = "DELETE FROM records WHERE name = '%s'" % remove_name
#results = db_powerdns.query(query)
pass
def add_record(name,zone,content,type='A',ttl=300):
# get zone database id
query = "SELECT id FROM domains WHERE name = '%s'" % zone
results = db_dns.query(query)
domain_id = results['results'][0]['id']
query = "SELECT * FROM records WHERE name='%s.%s' AND content='%s'" % (name,zone,content)
def kill_process(self):
"""
杀死当前的进程和sniff进程
"""
own = os.getpid()
result = os.popen('ps aux')
res = result.read()
for line in res.splitlines():
if 'jspider' in line:
pid = int(line.split(None,2)[1])
if pid != own:
os.kill(pid,signal.SIGKILL)
os.kill(own,signal.SIGKILL)
def insert_url(self,table_name,url,types,scheme,body):
"""
数据入库
"""
if types == "GET":
result = db().query("select count(*) from {table_name} where scheme = '{scheme}'".format(table_name=table_name,scheme=scheme))
if result == (0L,):
print G+'[INFO] [URL] {url} [{types}] {W}'.format(types=types,url=url,W=W)
db().update("insert into {table_name}(url,types,scheme,time) values('{url}','{types}','{scheme}','{time_}')".format(table_name=table_name,url=url,types=types,scheme=scheme,time_=time.time()))
elif types == "POST":
result = db().query("select count(*) from {table_name} where scheme = '{scheme}'".format(table_name=table_name,scheme=scheme))
if result == (0L,):
print G+'[INFO] [URL] {url} [DATA]{body} [{types}] {W}'.format(types=types,url=url,body=body,W=W)
db().update("insert into {table_name}(url,types,body,scheme,time) values('{url}','{types}','{body}','{scheme}','{time_}')".format(table_name=table_name,url=url,types=types,body=body,scheme=scheme,time_=time.time()))
else:
print "[ERROR] Error in data storage!"