async def timing():
"""A simple polling timing program
"""
# global count
cache["count"] += 1
if cache["count"] >= 3:
for i in Host.select():
if i.id_ in cache:
if cache[i.id_].target != "http://{}:{}".format(
i.host, i.port):
cache[i.id_] = scrapyd_object(i)
else:
cache[i.id_] = scrapyd_object(i)
cache["count"] = 0
if not cache:
return
timing_info = await app.get(Timing.select())
if not timing_info:
return
for i in timing_info:
if time.time() - i.last_time >= i.run_time:
try:
await cache[i.host_id].schedule(project=i.project_name,
spider=i.spider_name)
i.last_time = time.time()
await app.update(i, only=[Timing.last_time])
# count += 1
# print(count)
# app_log.info("runing timing task ok: {}".format(i.spider_name))
except Exception as f:
app_log.error(f)
async def get(self):
host_info = await self.application.objects.get(Host.select())
if not host_info:
self.write({
"data": False,
"msg": "server error!"
},
status_code=500)
return []
data = ModelsToDict(data=host_info,
field_name={'id_': 'id'},
shield_field=[Host.scrapyd_password],
field_handle={Host.create_time: self.time2time})
self.write({"data": data})
cache scrapy_api object. cache time : 60
"""
if not host_info:
return None
if ismodels is True:
return ScrapyApi(target="http://{}:{}".format(host_info.host,
host_info.port),
timeout=timeout)
return ScrapyApi(target="http://{}:{}".format(host_info.host,
host_info.port),
timeout=timeout)
cache = {}
try:
for i in Host.select():
cache[i.id_] = scrapyd_object(i)
except:
pass
# count = 1
async def timing():
"""A simple polling timing program
"""
# global count
if not cache:
return
async def delete(self, host_id):
await self.application.objects.delete(Host.getOne(Host.id_ == host_id))
await self.application.objects.delete(
Timing.getOne(Timing.host_id == host_id))
self.write({"data": True})
def get(self):
_id = self.get_argument('id', 0)
Host.delete().where(Host.id_ == _id).execute()
self.write(self.put_jsonp({"msg": "OK"}))
async def get(self, *args, **kwargs):
query = await self.application.objects.get(Host.select())
self.render('host_list.html', item=models_to_dict(query))
def get(self, *args, **kwargs):
self.render('host_list.html', item=models_to_dict(Host.select()))