def download(self, dic, chunk_size=1024*4, progressBar=show_progress):
"""将取得的内容写入文件"""
try:
(resp, pos, size, total) = self.get_response(dic)
except TypeError as nt:
logger.error('%s', nt.reason)
bsf = pos
logger.debug('bsf: %d', bsf)
_filename = get_filename(dic['FILENAME'])
if bsf != 0:
fh = open(_filename, 'ab')
total = total+pos
logger.info('Resume mode: start from %d', pos)
elif bsf == 0:
fh = open(_filename, 'wb')
while True:
try:
buf = resp.read(chunk_size)
if not buf:
break
fh.write(buf)
bsf+=len(buf)
progressBar(total, bsf, dic['HASH'], self.queue.qsize())
except (urllib2.HTTPError, ssl.SSLError):
logger.error('\nTIMEOUT')
continue
fh.flush()
fh.close()
try:
sys.stdout.write('\n')
db_handle.record(dic)
except:
pass
def conn(self):
""" Create ssh connection.
启动服务.
Create ssh connection using parameters given to service instance,using module paramiko.
使用paramiko模块，通过service实例给定参数建立ssh连接。
Args:
参数：
ip,username,password
指定ssh超时时间timeout 5s
Returns:
返回值：
Return a ssh connection .
返回一个ssh连接。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=self.ip, username=self.username, password=self.password, timeout=5)
mylogger.debug("Connected to %s." % self.ip)
return ssh
except Exception as exceptions:
mylogger.error(exceptions)
def run(self):
try:
logger.debug('Thread %s started.', self.name)
while not self.queue.empty():
dic = self.queue.get()
self.download(dic)
else:
logger.debug('nothing to do, exit.')
self.thread_list.remove(self)
logger.debug('Wait 3 seconds.')
time.sleep(3)
except:
pass
finally:
# time.sleep(1)
logger.debug('End thread %s.', self.name)
logger.debug('Thread list contains: %s', self.thread_list)
def get_response(self, dic):
pos = get_file_size(get_filename(dic['FILENAME']))
logger.debug('Filename: %s', get_filename(dic['FILENAME']))
logger.debug('Position: %d', pos)
try:
req = urllib2.Request(dic['URL'])
if pos != 0:
req.headers['Range'] = 'bytes=%s-' % pos
resp = urllib2.urlopen(req, timeout=30)
totalsize = resp.info().getheader('Content-Length').strip()
totalsize = int(totalsize)
size = totalsize-pos
return (resp, pos, size, totalsize)
except (urllib2.HTTPError, ssl.SSLError, socket.timeout):
# handle timeout, nothing here
logger.error('%s', 'time out.')
except urllib2.URLError as e:
logger.error('%s', e.reason)
def check_service(self, sshclient):
""" Check service status.
使用传入连接，检查服务状态，以解决重复创建连接问题.
Checking service status by running scripts like db2check.sh on server.
通过执行服务器上的服务检查脚本，如db2check.sh，确定服务是否活动。
Args:
参数：
check,status
检查脚本，状态
Returns:
返回值：
Return boolean values.True while service running,False while not.
返回布尔类型值：True代表服务已启动，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
command = self.check
if command == "":
mylogger.error("CHECK ERROR:check command is null.")
return False
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
mylogger.debug(command)
mylogger.info(output)
# only way to return success is get keyword "_SUCCESS_"
if len(output) > 0:
if output.__contains__("_SUCCESS_"):
mylogger.info("CHECK %s [SUCCESS]!" % self.__str__())
return True
else:
mylogger.info("CHECK %s [FAILED]!" % self.__str__())
return False
if len(errs) > 0:
mylogger.error("CHECK ERROR:command error happened.")
mylogger.error(errs)
return False
except Exception as e:
mylogger.error(e)
def release(self, ssh):
""" Disconnect a ssh connection.
释放连接.
Release a ssh connection using close()
释放ssh连接
Args:
参数：
sshclient
Returns:
no return values
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh.close()
mylogger.debug("Disconnected from %s." % self.ip)
except Exception as exceptions:
mylogger.error(exceptions)
finally:
ssh.close()
def url_eater(self, queue):
"""
接受用户输入，将检测结果合法的元素放入队列中
"""
try:
while 1:
sys.stdout.write('Enter url: ')
tmp_url = sys.stdin.readline().strip('\n')
tmp_time = time.ctime()
if tmp_url == 'q':
logger.debug('Thread pool size: %d', len(self.thread_list))
if len(self.thread_list) == 0:
db_handle.disconnect()
sys.exit(0)
else:
logger.error('Thread Fetcher is working! Please retry until it done.')
time.sleep(3)
continue
if (tmp_url.find('http', 0) == -1 and tmp_url.find('://', 8) != -1):
logger.error('Invalid URL: Are you forget the head of http or enter multiple url here?')
break
if (tmp_url.split('.')[-1] != 'png' and tmp_url.split('.')[-1] != 'jpg'):
tmp = self.handle_link(tmp_url)
tmp_url = tmp
# logger.error('Invalid file extension: %s.', tmp_url.split(r'/')[-1])
# continue
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
logger.debug('WAIT@EATER.')
if len(self.thread_list) == 0:
fetcher = Fetcher('Link Downloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
if len(self.thread_list) == 0:
fetcher = Fetcher('URL doenloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
logger.debug('Pool of fetcher is full.')
except KeyboardInterrupt:
logger.debug('User aborted, but I will record recent downloads.')
db_handle.disconnect()
sys.exit(99)
finally:
logger.debug('Thread list contains: %s', thread_list)
time.sleep(1)
def stop_service(self):
""" stop service .
停止服务.
Stop service by running scripts like db2stop.sh on server.
通过执行服务器上的服务检查脚本，如db2stop.sh，停止服务。
Args:
参数：
sshclient, nohupstop, stop
Returns:
返回值：
Return boolean values.True while service stop successfully,False while not.
返回布尔类型值：True代表服务停止成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
command = self.stop
if command == "":
mylogger.info("Check is NULL! service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
return True
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
mylogger.info("Stopping %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output, errs = "", ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
mylogger.info(output)
mylogger.error(errs)
# 默认认为停服务成功
mylogger.info("Check is passed for stop, service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
self.release(sshclient)
return True
def start_service(self):
""" start service .
启动服务.
Start service by running scripts like db2start.sh on server.
通过执行服务器上的启动脚本，如db2start.sh，启动服务。
Args:
参数：
sshclient, nohupstart, start
ssh连接，是否为nohup启动方式:YES/NO，启动脚本:/path/startscript.sh
Returns:
返回值：
Return boolean values.True while service start successfully,False while not.
返回布尔类型值：True代表服务启动成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
command = self.start
if self.nohupstart == "YES":
mylogger.info("NOHUP Starting %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.get_transport().open_session()
mylogger.debug(command)
channel.exec_command(command)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
# if for loop ended without return ,means check failed.
mylogger.info("NOHUP start %s [FAILED]!" % self.__str__())
self.release(sshclient)
return False
else:
mylogger.info("Check is NO, service started success by default.")
mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
else:
"""
Common way startup:
simulate a shell, execute command, and exit shell return to python program
"""
mylogger.info("Common Starting %s ..." % self.__str__())
mylogger.debug(command)
srvid = self.srvid
if srvid.__contains__("DB2"):
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
else:
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output = ""
errs = ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
if len(output) > 0:
mylogger.info(output)
if len(errs) > 0:
mylogger.error(errs)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
else:
mylogger.info("Check is passed, service start considered success.")
mylogger.info("Common start %s [SUCCESS]..." % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
return False
finally:
mylogger.debug(self.get_separator("end"))
errs = stderr.read()
except Exception, e:
mylogger.error(e)
mylogger.info(output)
mylogger.error(errs)
# 默认认为停服务成功
mylogger.info("Check is passed for stop, service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
finally:
mylogger.debug(self.get_separator("end"))
@fn_timer_logger
def check_service(self, sshclient):
""" Check service status.
使用传入连接，检查服务状态，以解决重复创建连接问题.
Checking service status by running scripts like db2check.sh on server.
通过执行服务器上的服务检查脚本，如db2check.sh，确定服务是否活动。
Args:
参数：
check,status
检查脚本，状态
Returns:
返回值：
Return boolean values.True while service running,False while not.
返回布尔类型值：True代表服务已启动，False反之。