def main():
sqlmgr = SQLiteExt("./text.db", 0)
try:
create_table_sql = '''CREATE TABLE `search` (
`id` BIGINT NOT NULL,
`uid` BIGINT NULL,
`title` VARCHAR(128) NULL,
`text` VARCHAR(40960) NULL,
`created_at` INT NULL,
`retweet_count` INT NULL,
`reply_count` INT NULL,
`fav_count` INT NULL,
`retweet_id` INT NULL,
`type` INT NULL,
`source_link` VARCHAR(256),
`edited_at` INT NULL,
`pic` VARCHAR(256) NULL,
`target` VARCHAR(256) NULL,
`source` VARCHAR(256) NULL,
PRIMARY KEY (`id`));'''
sqlmgr.create_table(create_table_sql)
except Exception, e:
mlog.log().error("Create table failed")
def create_table(self, sql):
if sql is not None and sql != '':
cur = self.__get_cursor()
cur.execute(sql)
self.conn.commit()
self.__close_all(cur)
else:
mlog.log().error('the [{}] is empty or equal None!'.format(sql))
def drop_table(self, table):
"""如果表存在,则删除表，如果表中存在数据的时候，使用该
方法的时候要慎用！"""
if table is not None and table != '':
sql = 'DROP TABLE IF EXISTS ' + table
cur = self.__get_cursor()
cur.execute(sql)
self.conn.commit()
self.__close_all(cur)
else:
mlog.log().error('the [{}] is empty or equal None!'.format(table))
def save(self, sql, data):
'''插入数据'''
if sql is not None and sql != '':
if data is not None:
cur = self.__get_cursor()
for d in data:
cur.execute(sql, d)
self.conn.commit()
self.__close_all(cur)
else:
mlog.log().error('the [{}] is empty or equal None!'.format(sql))
def __conection_sql(self):
try:
self.conn = sqlite3.connect(self.name, self.timeout)
if os.path.exists(self.name) and os.path.isfile(self.name):
self.type = 0
else:
self.conn = sqlite3.connect(':memory:')
self.type = 1
except Exception, e:
mlog.log().error('sqlite3 error:%s' % e)
return
def __get_conn(self, path):
"""获取到数据库的连接对象，参数为数据库文件的绝对路径
如果传递的参数是存在，并且是文件，那么就返回硬盘上面改
路径下的数据库文件的连接对象；否则，返回内存中的数据接
连接对象"""
conn = sqlite3.connect(path)
if os.path.exists(path) and os.path.isfile(path):
mlog.log().info('硬盘上面:[{}]'.format(path))
return conn
else:
conn = None
mlog.log().info('内存上面:[:memory:]')
return sqlite3.connect(':memory:')
def fetch(self, sql):
queue = []
if sql is not None and sql != '':
cur = self.__get_cursor()
cur.execute(sql)
r = cur.fetchall()
if len(r) > 0:
for e in range(len(r)):
queue.append(r[e])
return queue
else:
mlog.log().error('the [{}] is empty or equal None!'.format(sql))
return None
def run(self):
"""
连接取数据
"""
while True:
consumer = KafkaConsumer(bootstrap_servers=self.host)
consumer.subscribe([self.coname])
for message in consumer:
try:
json_info = json.loads(message[6])
print json_info
#self.callback(json_info)
except Exception, e:
mlog.log().error(e)
def start(self):
"""
连接取数据
"""
while True:
i = 0
for message in self.consumer:
json_info = ""
try:
json_info = json.loads(message[6])
except Exception, e:
mlog.log().error(message)
if len(json_info) > 0:
mlog.log().debug(json_info)
self.callback(json_info)
def get(self, basic_path, filename, callback=None):
if not self.ping():
return False
try:
#self.ftp.cwd('~/text_storage')
path_list = basic_path.split('/')
for _path in path_list:
self.ftp.cwd(_path)
#self.ftp.cwd(basic_path)
file_size = self.ftp.size(filename)
if callback is None:
self.ftp.retrbinary('RETR ' + filename, self.callback,
file_size)
else:
self.ftp.retrbinary('RETR ' + filename, callback, file_size)
return True
except Exception, e:
mlog.log().error("ftp error:%s url:%s", e, filename)
return False
def __u_connect(self):
self.ftp.set_pasv(True, self.host)
try:
if not self.ftp.connect(self.host, self.port, self.timeout):
mlog.log().error("connect ftp server failed")
return False
if not self.ftp.login(self.name, self.pwd):
mlog.log().error("login ftp server failed")
return False
self.is_connected = True
mlog.log().info("host : " + self.host + " ftp login success")
return True
except Exception, e:
mlog.log().error("ftp error[%s]", e)
return False
def log(self):
mlog.log().info(self.host)
mlog.log().info(self.port)
mlog.log().info(self.name)
mlog.log().info(self.pwd)
def close(self):
self.ftp.close()
mlog.log().info("close ftp ip %s", self.host)