def __init__(self):
self.path_tpl_entity = PathUtil.root('templates') + '/entity.tpl'
self.path_tpl_mapper = PathUtil.root('templates') + '/mapper.tpl'
self.project_java = 'test'
self.mysql_util = MysqlUtil({
'ip': 'localhost',
'user': '******',
'passwd': '123456',
'db': 'tacomall'
})
self.all_tables = []
self._query_all_tables()
def __init__(self, root_path):
self.root_path = root_path
self.SELECT_SQL = "SELECT page_no, loc_url,status FROM illust WHERE illust_id={}"
self.RESET_ILLUSTER_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {}"
self.RESET_ILLUST_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {} AND illust_id NOT IN ({})"
self.FILTER_SQL = "SELECT illust_id FROM illust WHERE status = 10 AND illust_id in ({})"
# 设priority = 6代表已经检查完毕
self.MAKE_ILLUSTER_STATUS_DONE_SQL = "UPDATE illuster SET priority = 6 WHERE illuster_id = {}"
self.CHECK_IF_DONE = 'SELECT priority FROM illuster WHERE illuster_id={}'
self.GET_DONE_ILLUSTER = 'SELECT illuster_id FROM illuster WHERE priority=6'
self.db_util = MysqlUtil()
self.logger = Log(__name__, log_cate='checker').get_log()
self.before_illuster_id = None
def init_db():
conf = ConfigUtil()
host = conf.get_config(conf='host', section='db_info')
port = conf.get_config(conf='port', section='db_info')
username = conf.get_config(conf='username', section='db_info')
password = conf.get_config(conf='password', section='db_info')
database = conf.get_config(conf='database', section='db_info')
conn = MySQLdb.connect(host=host,
user=username,
passwd=password,
port=int(port))
cur = conn.cursor()
sql = 'CREATE DATABASE IF NOT EXISTS `{db}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;'.format(
db=database)
cur.execute(sql)
conn.commit()
cur.close()
conn.close()
session = MysqlUtil().get_session()
BaseModel.metadata.create_all(MysqlUtil().get_engine())
email = conf.get_config('email', 'admin_account')
if session.query(User).filter_by(email=email).first() is None:
password = conf.get_config('password', 'admin_account')
username = conf.get_config('username', 'admin_account')
user = User(username=username,
password=generate_password_hash(password),
email=email,
status=1,
is_admin=1,
register_time=datetime.datetime.now(),
last_login_time=datetime.datetime.now())
session.add(user)
session.commit()
def __init__(self):
self.session = MysqlUtil().get_session()
def main():
is_new = input('是否重新导出数据？ yes/no：')
version = time.time()
if is_new == 'no':
version = input('请输入历史版本 ps:1601260075.6850908：')
if not os.path.exists('./.sql/dump/{0}'.format(version)):
raise Exception('版本不存在')
path_dump = './.sql/dump/{0}'.format(version)
if not os.path.exists(path_dump):
os.makedirs(path_dump)
db_from_config = {
'ip': '',
'port': 3306,
'user': '',
'passwd': '',
'db': ''
}
db_to_config = {'ip': '', 'port': 3306, 'user': '', 'passwd': '', 'db': ''}
if 'rds' in db_to_config['ip']:
raise Exception('导入数据库存在敏感地址，请再次验证')
all_tables = []
mysql_util = MysqlUtil(config=db_from_config)
sql_all_tables = 'SHOW TABLES'
sql_all_tables_result = mysql_util.query_sql(sql_all_tables)
for table in list(sql_all_tables_result):
all_tables.append(table[0])
if is_new == 'yes':
for t in all_tables:
print('----> from database <{0}>'.format(db_from_config['ip']))
print('----> dump table <{0}>'.format(t))
cmd_dump_sql = 'mysqldump --set-gtid-purged=off -h{h} -P{P} -u{u} -p{p} {db} --tables {tb}>{path_dump}/{tb}.sql'.format(
h=db_from_config['ip'],
P=db_from_config['port'],
u=db_from_config['user'],
p=db_from_config['passwd'],
db=db_from_config['db'],
path_dump=path_dump,
tb=t)
execute_cmd(cmd_dump_sql)
verify_code = ''.join(
random.sample([
'z', 'y', 'x', 'w', 'v', 'u', 't', 's', 'r', 'q', 'p', 'o', 'n',
'm', 'l', 'k', 'j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'
], 5))
print('----------WARNING START----------')
print('----------infomation below must be readed carefully!----------')
print(
'----------or it will make a unexpectable and harmful issue----------')
print('----------FROM DATABASE INFO----------')
print('----------IP {0}----------'.format(db_from_config['ip']))
print('----------DATABASE {0}----------'.format(db_from_config['db']))
print('----------TO DATABASE INFO----------')
print('----------IP {0}----------'.format(db_to_config['ip']))
print('----------DATABASE {0}----------'.format(db_to_config['db']))
print('----------WARNING END----------')
print('----------VERIFY CODE: {0}------'.format(verify_code))
is_confirm_do = input('已阅读警告 yes/no：')
if is_confirm_do == 'no':
print('程序退出')
return
if is_confirm_do == 'yes':
check_verify_code = input('输入上述验证码：')
if check_verify_code != verify_code:
raise Exception('验证码错误')
else:
for t in all_tables:
print('----> to database <{0}>'.format(db_to_config['ip']))
print('----> import table <{0}>'.format(t))
cmd_import_sql = 'mysql -h{h} -P{P} -u{u} -p{p} {db}<{path_dump}/{tb}.sql'.format(
h=db_to_config['ip'],
P=db_to_config['port'],
u=db_to_config['user'],
p=db_to_config['passwd'],
db=db_to_config['db'],
path_dump=path_dump,
tb=t)
execute_cmd(cmd_import_sql)
#!/usr/bin/env python # coding=UTF-8 ''' # Desc： # Author：TavisD # Time：2016-10-10 10:53 # Ver：V1.0 ''' from utils.api_client import APIClient from utils.file_util import FileUtil from utils.mysql_util import MysqlUtil from utils.gen_util import GenUtil api_client = APIClient() file_util = FileUtil() mysql_util = MysqlUtil() gen_util = GenUtil()
def __init__(self):
user_dao = MysqlUtil()
self.session = user_dao.get_session()
def __init__(self, path):
self.db_util = MysqlUtil()
self.spdier_util = SpiderUtil()
self.root_path = path
self.logger = Log(__name__).get_log()
# |\ __ \|\ \|\ \|\___ ___\\ __ \
# \ \ \|\ \ \ \\\ \|___ \ \_\ \ \|\ \
# \ \ __ \ \ \\\ \ \ \ \ \ \ \\\ \
# \ \ \ \ \ \ \\\ \ \ \ \ \ \ \\\ \
# \ \__\ \__\ \_______\ \ \__\ \ \_______\
# \|__|\|__|\|_______| \|__| \|_______|
# ________ ________ ________ ___ __ ___ _______ ________
# |\ ____\|\ __ \|\ __ \|\ \ |\ \|\ \ |\ ___ \ |\ __ \
# \ \ \___|\ \ \|\ \ \ \|\ \ \ \ \ \ \ \ \ \ \ __/|\ \ \|\ \
# \ \ \ \ \ _ _\ \ __ \ \ \ __\ \ \ \ \ \ \ \_|/_\ \ _ _\
# \ \ \____\ \ \\ \\ \ \ \ \ \ \|\__\_\ \ \ \____\ \ \_|\ \ \ \\ \|
# \ \_______\ \__\\ _\\ \__\ \__\ \____________\ \_______\ \_______\ \__\\ _\
# \|_______|\|__|\|__|\|__|\|__|\|____________|\|_______|\|_______|\|__|\|__|
conn = MysqlUtil()
#tags = conn.get_all('SELECT * FROM `test1.0`.`tag_Asyn`')
ids = conn.get_all(
'SELECT `test1.0`.`book_Asyn`.`book_id` FROM `test1.0`.`book_Asyn` LEFT JOIN `test1.0`.`book_detail` ON `test1.0`.`book_Asyn`.`book_id` = `test1.0`.`book_detail`.`book_id` WHERE `book_introduct` IS NULL;'
)
print(ids)
#cids = conn.get_all(' SELECT `test1.0`.`book_Asyn`.`book_id` FROM `test1.0`.`book_Asyn` LEFT JOIN `commenttable_Asyn` cA on `book_Asyn`.`book_id` = cA.`book_id` WHERE `comment` IS NULL; ')
runner = CrawlerRunner(get_project_settings())
@defer.inlineCallbacks
def crawl():
while True:
for tag in tags:
print('*********************')
print('\t\a' + tag[0])