def close_task(task_pool_id):
agent_name = os.environ['agent_name']
current_time = datetime.datetime.now()
session.query(TaskItem).filter(TaskItem.id == task_pool_id).update(
{TaskItem.status: "done",
TaskItem.processor: agent_name,
TaskItem.last_update: current_time})
def close_task(task_id):
current_time = datetime.datetime.now()
session.query(TrackingTask).filter(TrackingTask.id == task_id).update({
TrackingTask.status:
'done',
TrackingTask.end_time:
current_time,
TrackingTask.last_update:
current_time
})
def get_pending_task(batch_num):
agent_name = os.environ['agent_name']
job = os.environ['job']
if job == "bc":
task_pools = session.query(TaskItem).filter(TaskItem.batch_num == batch_num).filter(
TaskItem.status == 'pending', TaskItem.type == 'bc')
else:
task_pools = session.query(TaskItem).filter(TaskItem.batch_num == batch_num).filter(
TaskItem.status == 'pending', TaskItem.type != 'bc')
if task_pools.count() == 0:
logger.info("本Agent没有待巡检任务，Agent切换为waiting状态: %s", agent_name)
# 没有pending状态的任务
gl.set_value('STATUS', False)
return None, None
else:
logger.info("%s 准备执行可以处理的任务,倒数第：%s 个...", agent_name, str(task_pools.count()))
task_pool = task_pools.first()
session.query(TaskItem).filter(TaskItem.id == task_pool.id).update({"status": "processing"})
if task_pool.type == "weburl":
logger.info("task_pool.website_id：%s", task_pool.website_id)
weburl = session.query(Weburl).filter(Weburl.url == task_pool.url).filter(
Weburl.website_id == task_pool.website_id).all()
if len(weburl):
return weburl[0], task_pool
else:
logger.info("task_pool.website_id：%s", task_pool.website_id)
logger.info("task_pool.id：%s", task_pool.id)
session.query(TaskItem).filter(TaskItem.id == task_pool.id).update({"status": "done"})
return None, None
else:
website = session.query(Website).filter(Website.id == task_pool.website_id).one()
return website, task_pool
def add(weburl):
logger.info("add weburl to db: %s", weburl.url)
engine = create_engine(
'mysql://%s:%s@%s/%s?charset=utf8&autocommit=true' %
(username, password, host, database),
encoding='utf-8',
echo=False,
pool_size=100,
pool_recycle=10)
Session = sessionmaker(bind=engine)
session = Session()
try:
exist_weburl = session.query(Weburl).filter(
Weburl.url == weburl.url).filter(
Weburl.website_id == weburl.website_id).all()
if len(exist_weburl):
pass
else:
weburl.create_time = datetime.datetime.now()
weburl.last_update = datetime.datetime.now()
session.add(weburl)
session.commit()
except Exception as e:
print(e)
session.rollback()
raise
finally:
session.close()
def get_by_name(name):
third_config = session.query(ThirdConfig).filter(
ThirdConfig.name == name).all()
if len(third_config):
return third_config[0].cookie
else:
return "nocookie"
class TestMysql(object):
if __name__ == "__main__":
# 创建新User对象:
new_config = Global(
level='Bob',
freq='1',
create_time=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# 添加到session:
session.add(new_config)
# 提交即保存到数据库:
config1 = session.query(Global).filter(Global.id == 4).one()
print(config1.level)
config2 = session.query(Global).filter(Global.level == 'Bob').all()
for i in config2:
print(i.id, i.level)
session.query(Global).filter(Global.id == 4).update({"level": "1099"})
# 关闭session:
session.close()
def update(tracking_detail):
session.query(TrackingDetail).filter(
TrackingDetail.id == tracking_detail.id).update({
TrackingDetail.status:
tracking_detail.status,
TrackingDetail.start_time:
tracking_detail.start_time,
TrackingDetail.end_time:
tracking_detail.end_time,
TrackingDetail.url:
tracking_detail.url,
TrackingDetail.des:
tracking_detail.des,
TrackingDetail.snapshot:
tracking_detail.snapshot,
TrackingDetail.result:
tracking_detail.result,
TrackingDetail.retry:
tracking_detail.retry + 1
})
def get_overtime():
filterd = []
website_ids = session.query(distinct(Weburl.website_id)).all()
websites = session.query(Website).all()
strategy_service = StrategyService()
strategy = strategy_service.get_strategy()
now = int(round(time.time() * 1000))
overtime = now - strategy.cache_days * 24 * 60 * 60 * 1000
for website in websites:
last_gather_time = website.last_gather_time
if last_gather_time is None:
filterd.append(website)
continue
elif int(round(time.mktime(time.strptime(str(last_gather_time), "%Y-%m-%d %H:%M:%S")) * 1000)) > overtime:
if tuple([website.id]) not in website_ids:
filterd.append(website)
else:
continue
else:
filterd.append(website)
return filterd
def get_websites(task_id):
filterd = []
inspect_dao = InspectTaskDao()
inspect_task = inspect_dao.get_task(task_id)
websites = session.query(Website).filter(
Website.attention == inspect_task.attention).filter(
Website.industry.contains(inspect_task.industry)).filter(
Website.industry2.contains(inspect_task.industry2)).all()
strategy_service = StrategyService()
strategy = strategy_service.get_strategy()
now = time.time()
overtime = now - strategy.cache_days * 24 * 60 * 1000
for website in websites:
last_gather_time = website.last_gather_time
if last_gather_time is None:
continue
elif time.mktime(
time.strptime(str(last_gather_time),
"%Y-%m-%d %H:%M:%S")) > overtime:
continue
else:
filterd.append(website)
return filterd
def get_task(task_id):
inspect_task = session.query(InspectTask).filter(
InspectTask.id == task_id).one()
return inspect_task
def get_all():
websites = session.query(Website).all()
return websites
def get_all():
keywords = session.query(Keyword).all()
return keywords
def get_by_task(task_id, status):
tracking_details = session.query(TrackingDetail).filter(
TrackingDetail.task_id == task_id,
TrackingDetail.status == status).all()
return tracking_details
def get_all():
weburl = session.query(Weburl).all()
return weburl
def get_by_name(website_name):
website = session.query(Website).filter(Website.website_name == website_name).one()
return website
def get_by_merchant(merchant_name):
website = session.query(Website).filter(Website.merchant_name == merchant_name).one()
return website
def get_by_id(website_id):
website = session.query(Website).filter(Website.id == website_id).one()
return website
def get_all():
platforms = session.query(ThirdPlatform).all()
return platforms
def get_strategy():
strategy = session.query(Strategy).one()
return strategy