def download(self, dic, chunk_size=1024*4, progressBar=show_progress):
"""将取得的内容写入文件"""
try:
(resp, pos, size, total) = self.get_response(dic)
except TypeError as nt:
logger.error('%s', nt.reason)
bsf = pos
logger.debug('bsf: %d', bsf)
_filename = get_filename(dic['FILENAME'])
if bsf != 0:
fh = open(_filename, 'ab')
total = total+pos
logger.info('Resume mode: start from %d', pos)
elif bsf == 0:
fh = open(_filename, 'wb')
while True:
try:
buf = resp.read(chunk_size)
if not buf:
break
fh.write(buf)
bsf+=len(buf)
progressBar(total, bsf, dic['HASH'], self.queue.qsize())
except (urllib2.HTTPError, ssl.SSLError):
logger.error('\nTIMEOUT')
continue
fh.flush()
fh.close()
try:
sys.stdout.write('\n')
db_handle.record(dic)
except:
pass
def insert_item(self, data):
try:
mylogger.info('start to insert by using isert_item function')
db = self.db
coll = db["Computer_Science"]
coll.insert(data)
mylogger.info('paper {} get its categories'.format(data['id']))
except Exception as e:
print(e)
def set_socks5_proxy():
status = cfg_reader.get_cfg('proxy', 'enabled')
serv = cfg_reader.get_cfg('proxy', 'server')
pt = int(cfg_reader.get_cfg('proxy', 'port'))
if status == '1':
logger.info('Use proxy %s:%s' % (serv, pt))
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, serv, pt)
socket.socket = socks.socksocket
else:
return
def dostart(srvid, ip, username, password, nohupstart, start, nohupstop, stop,
needcheck, check, status):
service = Service(srvid, ip, username, password, nohupstart, start,
nohupstop, stop, needcheck, check, status)
if service.start_service():
mylogger.info(
"[------------ %s start [success]. Going to next service >>>>>>>>>>]"
% service.__str__())
return True
else:
mylogger.error(
"[------------ %s start [failed]. Workflow terminated! XXXXXXXXXX]"
% service.__str__())
return False
def handle_link(self, link):
# 因为有些网址里含有CJK字符，所以在这个里边需要进行一下转码
link = unicode(link)
req = urllib2.Request(link, headers={'User-Agent': UA})
resp = urllib2.urlopen(req)
cont = resp.read()
soup = BeautifulSoup(cont)
try:
resu = soup.find_all('a', class_='original-file-unchanged')
url = resu[0].get('href')
except IndexError as ie:
logger.error('Lossless version not found, download changed version.')
resu = soup.find_all('a', class_='original-file-changed')
url = resu[0].get('href')
# result = PATTERN.match(cont).group(1)
# logger.info('Matched: %s', result)
logger.info('Matched: %s', url)
return url
def docheck(srvid, ip, username, password, nohupstart, start, nohupstop, stop,
needcheck, check, status):
service = Service(srvid, ip, username, password, nohupstart, start,
nohupstop, stop, needcheck, check, status)
mylogger.info("-" * 105)
mylogger.info(":[[ -" + service.check_service1() + "- ]] : " +
service.__str__() + "")
mylogger.info("-" * 105)
def check_service(self, sshclient):
""" Check service status.
使用传入连接，检查服务状态，以解决重复创建连接问题.
Checking service status by running scripts like db2check.sh on server.
通过执行服务器上的服务检查脚本，如db2check.sh，确定服务是否活动。
Args:
参数：
check,status
检查脚本，状态
Returns:
返回值：
Return boolean values.True while service running,False while not.
返回布尔类型值：True代表服务已启动，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
command = self.check
if command == "":
mylogger.error("CHECK ERROR:check command is null.")
return False
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
mylogger.debug(command)
mylogger.info(output)
# only way to return success is get keyword "_SUCCESS_"
if len(output) > 0:
if output.__contains__("_SUCCESS_"):
mylogger.info("CHECK %s [SUCCESS]!" % self.__str__())
return True
else:
mylogger.info("CHECK %s [FAILED]!" % self.__str__())
return False
if len(errs) > 0:
mylogger.error("CHECK ERROR:command error happened.")
mylogger.error(errs)
return False
except Exception as e:
mylogger.error(e)
def url_eater(self, queue):
"""
接受用户输入，将检测结果合法的元素放入队列中
"""
try:
while 1:
sys.stdout.write('Enter url: ')
tmp_url = sys.stdin.readline().strip('\n')
tmp_time = time.ctime()
if tmp_url == 'q':
logger.debug('Thread pool size: %d', len(self.thread_list))
if len(self.thread_list) == 0:
db_handle.disconnect()
sys.exit(0)
else:
logger.error('Thread Fetcher is working! Please retry until it done.')
time.sleep(3)
continue
if (tmp_url.find('http', 0) == -1 and tmp_url.find('://', 8) != -1):
logger.error('Invalid URL: Are you forget the head of http or enter multiple url here?')
break
if (tmp_url.split('.')[-1] != 'png' and tmp_url.split('.')[-1] != 'jpg'):
tmp = self.handle_link(tmp_url)
tmp_url = tmp
# logger.error('Invalid file extension: %s.', tmp_url.split(r'/')[-1])
# continue
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
logger.debug('WAIT@EATER.')
if len(self.thread_list) == 0:
fetcher = Fetcher('Link Downloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
if len(self.thread_list) == 0:
fetcher = Fetcher('URL doenloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
logger.debug('Pool of fetcher is full.')
except KeyboardInterrupt:
logger.debug('User aborted, but I will record recent downloads.')
db_handle.disconnect()
sys.exit(99)
finally:
logger.debug('Thread list contains: %s', thread_list)
time.sleep(1)
def stop_service(self):
""" stop service .
停止服务.
Stop service by running scripts like db2stop.sh on server.
通过执行服务器上的服务检查脚本，如db2stop.sh，停止服务。
Args:
参数：
sshclient, nohupstop, stop
Returns:
返回值：
Return boolean values.True while service stop successfully,False while not.
返回布尔类型值：True代表服务停止成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
command = self.stop
if command == "":
mylogger.info("Check is NULL! service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
return True
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
mylogger.info("Stopping %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output, errs = "", ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
mylogger.info(output)
mylogger.error(errs)
# 默认认为停服务成功
mylogger.info("Check is passed for stop, service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
self.release(sshclient)
return True
def start_service(self):
""" start service .
启动服务.
Start service by running scripts like db2start.sh on server.
通过执行服务器上的启动脚本，如db2start.sh，启动服务。
Args:
参数：
sshclient, nohupstart, start
ssh连接，是否为nohup启动方式:YES/NO，启动脚本:/path/startscript.sh
Returns:
返回值：
Return boolean values.True while service start successfully,False while not.
返回布尔类型值：True代表服务启动成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
command = self.start
if self.nohupstart == "YES":
mylogger.info("NOHUP Starting %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.get_transport().open_session()
mylogger.debug(command)
channel.exec_command(command)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
# if for loop ended without return ,means check failed.
mylogger.info("NOHUP start %s [FAILED]!" % self.__str__())
self.release(sshclient)
return False
else:
mylogger.info("Check is NO, service started success by default.")
mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
else:
"""
Common way startup:
simulate a shell, execute command, and exit shell return to python program
"""
mylogger.info("Common Starting %s ..." % self.__str__())
mylogger.debug(command)
srvid = self.srvid
if srvid.__contains__("DB2"):
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
else:
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output = ""
errs = ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
if len(output) > 0:
mylogger.info(output)
if len(errs) > 0:
mylogger.error(errs)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
else:
mylogger.info("Check is passed, service start considered success.")
mylogger.info("Common start %s [SUCCESS]..." % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
return False
finally:
mylogger.debug(self.get_separator("end"))
def main():
# program run only by devops
if getpass.getuser() != "devops":
mylogger.error("This program should only be run by devops!")
exit(99)
command = ""
for arg in sys.argv:
command = command + " " + arg
mylogger.info(get_separator(command, "line"))
mylogger.info(get_separator(command, "begin"))
mylogger.info(get_separator(command, "line"))
try:
system, operation, service1, service2 = get_args()
service_list = get_json(system)
if service1 == "" and service2 == "":
if operation == "start":
for service in service_list:
if dostart(**service):
pass
else:
break
elif operation == "stop":
service_list.reverse()
for service in service_list:
if dostop(**service):
pass
else:
break
elif operation == "check":
for service in service_list:
docheck(**service)
elif operation == "list":
for service in service_list:
mylogger.info("[ " + service["srvid"] + " ]" + ": " +
service["start"])
elif service2 == "":
# start one service by id
if operation == "start":
for service in service_list:
if service["srvid"] == service1:
dostart(**service)
elif operation == "stop":
for service in service_list:
if service["srvid"] == service1:
dostop(**service)
elif operation == "check":
for service in service_list:
if service["srvid"] == service1:
docheck(**service)
else:
idlist = get_srv_ids(system)
i = idlist.index(service1)
j = idlist.index(service2) + 1
service_list1 = service_list.__getslice__(i, j)
if operation == "start":
for service in service_list1:
if dostart(**service):
pass
else:
break
elif operation == "stop":
service_list1.reverse()
for service in service_list1:
if dostop(**service):
pass
else:
break
elif operation == "check":
for service in service_list1:
docheck(**service)
finally:
mylogger.info(get_separator(command, "line"))
mylogger.info(get_separator(command, "end"))
mylogger.info(get_separator(command, "line"))