def crawl_deal(self):
'''
crawl house deals which is crawl least
'''
continue_flag = args.arg().c
mdb = db.db()
index = 0
communitys = mdb.get_communitys(continue_flag ,index,10)
while communitys and len(communitys) > 0:
for c in communitys:
urlptn = "http://bj.lianjia.com/chengjiao/c%s/"
code = c['code']
dd = dealcrawler.DealCrawlerT('list', urlptn, code)
dd.start()
dd.join()
if dd.state() == 'ok':
mdb.inc_community_crawl(code)
else:
self.__log.warning('skip deal inc crawl for state(%s) not ok.', dd.state())
index += 10
communitys = mdb.get_communitys(continue_flag ,index,10)
print 'finished crawl deal with index %d' % index
def crawl_community(self):
'''
crawl communitys which is crawl least
'''
continue_flag = args.arg().c
mdb = db.db()
index = 0
districts = mdb.get_districts(continue_flag , index,10)
while len(districts) > 0:
for d in districts:
urlptn = "http://bj.lianjia.com/xiaoqu/%s/"
name = d['path']
cc = communitycrawler.CommunityCrawlerT('list', urlptn, name)
cc.start()
cc.join()
if cc.state() == 'ok':
mdb.inc_district_crawl(name)
else:
self.__log.warning('skip community inc crawl for state(%s) not ok.', cc.state())
index += 10
districts = mdb.get_districts(continue_flag , index,10)
def __init__(self, tp , urlptn, data, timeout = 10, interval = 10):
"""
init Crawler instance
args:
type: list / info
"""
super(DealCrawlerT, self).__init__(urlptn,data,timeout,interval)
self._type = tp
self._arg = args.arg()
def __init__(self):
"""
spider main entrance
"""
skip = args.arg().s
while 1 :
if False == skip:
self.crawl_community()
self.crawl_deal()
self.__log.info('crawler finished once at %s.' % datetime.datetime.now().strftime('%m-%d %H:%M'))
self.wait_untill('2')
def __init__(self):
"""
creates a child process, which returns. the parent
process manage the spider which may waits for a
KeyboardInterrupt and then kills
the child process using child pid.
"""
deamon = args.arg().d
try:
self._child = os.fork()
except OSError:
sys.exit("Unable to create new process.")
if self._child == 0:
# new process
MasterP._log.debug('new process runing...')
return
else:
if deamon == True:
self._run_in_deamon(self._child)
else:
self._wait_interrupt()
# -*- coding: utf-8 -*-
from args import arg
from zabbix import Zabbix
from readExcel import readExcel
zabbix = Zabbix()
if __name__ == '__main__':
a = arg('hello')
post_dict = a.all_args()
zabbix.login(post_dict)
function = str(post_dict['function'])
if post_dict.get('file', None):
excelDataList = readExcel(post_dict['file'])
if function in a.base_functions:
for _ in excelDataList:
print(zabbix.baseFunction(_))
if function in a.new_functions:
for _ in excelDataList:
globals()[function](zabbix, _)
else:
print(zabbix.baseFunction(post_dict))
"and these people are using it, "
"but it is just a part of the system they use. "
"Linux is the kernel: the program in the system that allocates the machine's "
"resources to the other programs that you run. "
"The kernel is an essential part of an operating system, "
"but useless by itself; "
"it can only function in the context of a complete operating system. "
"Linux is normally used in combination with the GNU operating system: "
"the whole system is basically GNU with Linux added, or GNU/Linux. "
'All the so-called "Linux" distributions are really distributions of GNU/Linux.'
)
@app.cmd(
"timer",
args=[arg("duration", type=int, help="duration in seconds")],
help="set an egg timer",
)
async def timer(message: Message) -> str:
if message.args.duration > 600:
return "duration cannot be more than 600s"
elif message.args.duration < 0:
return "duration cannot be less than 0s"
await asyncio.sleep(message.args.duration)
return f"Your {message.args.duration}s timer is up @{str(message.user.name)}!"
@app.cmd(
"spam",
args=[arg("text", type=str, help="text to spam")],