def download(self, dic, chunk_size=1024*4, progressBar=show_progress): """将取得的内容写入文件""" try: (resp, pos, size, total) = self.get_response(dic) except TypeError as nt: logger.error('%s', nt.reason) bsf = pos logger.debug('bsf: %d', bsf) _filename = get_filename(dic['FILENAME']) if bsf != 0: fh = open(_filename, 'ab') total = total+pos logger.info('Resume mode: start from %d', pos) elif bsf == 0: fh = open(_filename, 'wb') while True: try: buf = resp.read(chunk_size) if not buf: break fh.write(buf) bsf+=len(buf) progressBar(total, bsf, dic['HASH'], self.queue.qsize()) except (urllib2.HTTPError, ssl.SSLError): logger.error('\nTIMEOUT') continue fh.flush() fh.close() try: sys.stdout.write('\n') db_handle.record(dic) except: pass
def conn(self): """ Create ssh connection. 启动服务. Create ssh connection using parameters given to service instance,using module paramiko. 使用paramiko模块,通过service实例给定参数建立ssh连接。 Args: 参数: ip,username,password 指定ssh超时时间timeout 5s Returns: 返回值: Return a ssh connection . 返回一个ssh连接。 Raises: exceptions 发生异常时时记录异常信息 """ try: ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=self.ip, username=self.username, password=self.password, timeout=5) mylogger.debug("Connected to %s." % self.ip) return ssh except Exception as exceptions: mylogger.error(exceptions)
def disconnect(self): try: conn.commit() cur.close() conn.close() except sqlite3.DatabaseError, e: if isinstance(e.args[0], sqlite3.DatabaseError): logger.error('%s', e.args[0]) else: raise
def get_args(): arg_length = sys.argv.__len__() system, operation, service1, service2 = "", "", "", "" if arg_length not in (3, 4, 5): print_usage() exit(1) elif arg_length == 3: # qs start | stop | check | list system = sys.argv[1] operation = sys.argv[2] if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank", "wailian", "3par") or operation not in ("start", "stop", "check", "list"): print_usage() exit(1) elif arg_length == 4: # qs start | stop | check service1 system = sys.argv[1] operation = sys.argv[2] service1 = sys.argv[3] if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank", "wailian", "3par") or operation not in ("start", "stop", "check"): print_usage() exit(1) if not get_srv_ids(system).__contains__(service1): print_usage() exit(1) elif arg_length == 5: # qs start | stop | check service1 service2 system = sys.argv[1] operation = sys.argv[2] service1 = sys.argv[3] service2 = sys.argv[4] if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank", "wailian", "3par") or operation not in ("start", "stop", "check"): print_usage() exit(1) idlist = get_srv_ids(system) if not idlist.__contains__(service1) or not idlist.__contains__( service2): print_usage() exit(1) if idlist.index(service1) > idlist.index(service2): print_usage() mylogger.error( "PARAMETER ERROR: service2 should be after service1 !!!") exit(1) return system, operation, service1, service2
def get_json(system): try: json_name = system + "_workflow.json" if os.path.exists(json_name): f = codecs.open(json_name, mode='r', encoding='utf-8') service_list = json.load(f) f.close() return service_list else: mylogger.error("Json file not found.") exit(1) except Exception as exp: mylogger.error(exp) exit(1)
def print_usage(): command = "" for arg in sys.argv: command = command + " " + arg mylogger.error("PARAMETER ERROR: %s " % command) mylogger.error( "COMMAND FORMAT: python auto.py [system] [operation] [service1] [service2]" ) mylogger.error( " -- system: qs | sambank | ib | ndp | zf | zh | hg | ra | wailian " ) mylogger.error(" -- operation: start | stop | check | list") mylogger.error( " -- service1/service2: Get serviceid using 'list' argument" )
def dostart(srvid, ip, username, password, nohupstart, start, nohupstop, stop, needcheck, check, status): service = Service(srvid, ip, username, password, nohupstart, start, nohupstop, stop, needcheck, check, status) if service.start_service(): mylogger.info( "[------------ %s start [success]. Going to next service >>>>>>>>>>]" % service.__str__()) return True else: mylogger.error( "[------------ %s start [failed]. Workflow terminated! XXXXXXXXXX]" % service.__str__()) return False
def get_srv_ids(system): try: json_name = system + "_workflow.json" if os.path.exists(json_name): f = codecs.open(json_name, mode='r', encoding='utf-8') service_list = json.load(f) f.close() srvids = [] for service in service_list: srvids.append(service["srvid"]) return srvids else: mylogger.error("Json file not found.") exit(1) except Exception as exp: mylogger.error(exp) exit(1)
def get_response(self, dic): pos = get_file_size(get_filename(dic['FILENAME'])) logger.debug('Filename: %s', get_filename(dic['FILENAME'])) logger.debug('Position: %d', pos) try: req = urllib2.Request(dic['URL']) if pos != 0: req.headers['Range'] = 'bytes=%s-' % pos resp = urllib2.urlopen(req, timeout=30) totalsize = resp.info().getheader('Content-Length').strip() totalsize = int(totalsize) size = totalsize-pos return (resp, pos, size, totalsize) except (urllib2.HTTPError, ssl.SSLError, socket.timeout): # handle timeout, nothing here logger.error('%s', 'time out.') except urllib2.URLError as e: logger.error('%s', e.reason)
def handle_link(self, link): # 因为有些网址里含有CJK字符,所以在这个里边需要进行一下转码 link = unicode(link) req = urllib2.Request(link, headers={'User-Agent': UA}) resp = urllib2.urlopen(req) cont = resp.read() soup = BeautifulSoup(cont) try: resu = soup.find_all('a', class_='original-file-unchanged') url = resu[0].get('href') except IndexError as ie: logger.error('Lossless version not found, download changed version.') resu = soup.find_all('a', class_='original-file-changed') url = resu[0].get('href') # result = PATTERN.match(cont).group(1) # logger.info('Matched: %s', result) logger.info('Matched: %s', url) return url
def release(self, ssh): """ Disconnect a ssh connection. 释放连接. Release a ssh connection using close() 释放ssh连接 Args: 参数: sshclient Returns: no return values Raises: exceptions 发生异常时时记录异常信息 """ try: ssh.close() mylogger.debug("Disconnected from %s." % self.ip) except Exception as exceptions: mylogger.error(exceptions) finally: ssh.close()
def check_service(self, sshclient): """ Check service status. 使用传入连接,检查服务状态,以解决重复创建连接问题. Checking service status by running scripts like db2check.sh on server. 通过执行服务器上的服务检查脚本,如db2check.sh,确定服务是否活动。 Args: 参数: check,status 检查脚本,状态 Returns: 返回值: Return boolean values.True while service running,False while not. 返回布尔类型值:True代表服务已启动,False反之。 Raises: exceptions 发生异常时时记录异常信息,返回False """ try: command = self.check if command == "": mylogger.error("CHECK ERROR:check command is null.") return False stdin, stdout, stderr = sshclient.exec_command(command) output = stdout.read() errs = stderr.read() mylogger.debug(command) mylogger.info(output) # only way to return success is get keyword "_SUCCESS_" if len(output) > 0: if output.__contains__("_SUCCESS_"): mylogger.info("CHECK %s [SUCCESS]!" % self.__str__()) return True else: mylogger.info("CHECK %s [FAILED]!" % self.__str__()) return False if len(errs) > 0: mylogger.error("CHECK ERROR:command error happened.") mylogger.error(errs) return False except Exception as e: mylogger.error(e)
def url_eater(self, queue): """ 接受用户输入,将检测结果合法的元素放入队列中 """ try: while 1: sys.stdout.write('Enter url: ') tmp_url = sys.stdin.readline().strip('\n') tmp_time = time.ctime() if tmp_url == 'q': logger.debug('Thread pool size: %d', len(self.thread_list)) if len(self.thread_list) == 0: db_handle.disconnect() sys.exit(0) else: logger.error('Thread Fetcher is working! Please retry until it done.') time.sleep(3) continue if (tmp_url.find('http', 0) == -1 and tmp_url.find('://', 8) != -1): logger.error('Invalid URL: Are you forget the head of http or enter multiple url here?') break if (tmp_url.split('.')[-1] != 'png' and tmp_url.split('.')[-1] != 'jpg'): tmp = self.handle_link(tmp_url) tmp_url = tmp # logger.error('Invalid file extension: %s.', tmp_url.split(r'/')[-1]) # continue tmp_dict = { 'URL': tmp_url, 'FILENAME': unquote(tmp_url.split(r'/')[-1]), 'HASH': tmp_url.split(r'/')[-2], 'APTIME': tmp_time, 'FLG': False } logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME']) completed = db_handle.query(tmp_dict['HASH']) if completed == 0: logger.info('%s already downloaded.', tmp_dict['HASH']) continue try: logger.debug('ACQUIRED BY EATER') queue.put(tmp_dict) except: pass finally: logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize()) logger.debug('WAIT@EATER.') if len(self.thread_list) == 0: fetcher = Fetcher('Link Downloader', self.queue, thread_list) fetcher.setDaemon(True) self.thread_list.append(fetcher) logger.debug('Thread list contains: %s', thread_list) fetcher.start() else: tmp_dict = { 'URL': tmp_url, 'FILENAME': unquote(tmp_url.split(r'/')[-1]), 'HASH': tmp_url.split(r'/')[-2], 'APTIME': tmp_time, 'FLG': False } logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME']) completed = db_handle.query(tmp_dict['HASH']) if completed == 0: logger.info('%s already downloaded.', tmp_dict['HASH']) continue try: logger.debug('ACQUIRED BY EATER') queue.put(tmp_dict) except: pass finally: logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize()) if len(self.thread_list) == 0: fetcher = Fetcher('URL doenloader', self.queue, thread_list) fetcher.setDaemon(True) self.thread_list.append(fetcher) logger.debug('Thread list contains: %s', thread_list) fetcher.start() else: logger.debug('Pool of fetcher is full.') except KeyboardInterrupt: logger.debug('User aborted, but I will record recent downloads.') db_handle.disconnect() sys.exit(99) finally: logger.debug('Thread list contains: %s', thread_list) time.sleep(1)
def main(): # program run only by devops if getpass.getuser() != "devops": mylogger.error("This program should only be run by devops!") exit(99) command = "" for arg in sys.argv: command = command + " " + arg mylogger.info(get_separator(command, "line")) mylogger.info(get_separator(command, "begin")) mylogger.info(get_separator(command, "line")) try: system, operation, service1, service2 = get_args() service_list = get_json(system) if service1 == "" and service2 == "": if operation == "start": for service in service_list: if dostart(**service): pass else: break elif operation == "stop": service_list.reverse() for service in service_list: if dostop(**service): pass else: break elif operation == "check": for service in service_list: docheck(**service) elif operation == "list": for service in service_list: mylogger.info("[ " + service["srvid"] + " ]" + ": " + service["start"]) elif service2 == "": # start one service by id if operation == "start": for service in service_list: if service["srvid"] == service1: dostart(**service) elif operation == "stop": for service in service_list: if service["srvid"] == service1: dostop(**service) elif operation == "check": for service in service_list: if service["srvid"] == service1: docheck(**service) else: idlist = get_srv_ids(system) i = idlist.index(service1) j = idlist.index(service2) + 1 service_list1 = service_list.__getslice__(i, j) if operation == "start": for service in service_list1: if dostart(**service): pass else: break elif operation == "stop": service_list1.reverse() for service in service_list1: if dostop(**service): pass else: break elif operation == "check": for service in service_list1: docheck(**service) finally: mylogger.info(get_separator(command, "line")) mylogger.info(get_separator(command, "end")) mylogger.info(get_separator(command, "line"))
def start_service(self): """ start service . 启动服务. Start service by running scripts like db2start.sh on server. 通过执行服务器上的启动脚本,如db2start.sh,启动服务。 Args: 参数: sshclient, nohupstart, start ssh连接,是否为nohup启动方式:YES/NO,启动脚本:/path/startscript.sh Returns: 返回值: Return boolean values.True while service start successfully,False while not. 返回布尔类型值:True代表服务启动成功,False反之。 Raises: exceptions 发生异常时时记录异常信息,返回False """ try: mylogger.debug(self.get_separator("begin")) sshclient = self.conn() command = self.start if self.nohupstart == "YES": mylogger.info("NOHUP Starting %s ..." % self.__str__()) mylogger.debug(command) channel = sshclient.get_transport().open_session() mylogger.debug(command) channel.exec_command(command) if self.needcheck == "YES": for i in range(15): if self.check_service(sshclient): mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True time.sleep(1) # if for loop ended without return ,means check failed. mylogger.info("NOHUP start %s [FAILED]!" % self.__str__()) self.release(sshclient) return False else: mylogger.info("Check is NO, service started success by default.") mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True else: """ Common way startup: simulate a shell, execute command, and exit shell return to python program """ mylogger.info("Common Starting %s ..." % self.__str__()) mylogger.debug(command) srvid = self.srvid if srvid.__contains__("DB2"): stdin, stdout, stderr = sshclient.exec_command(command) output = stdout.read() errs = stderr.read() else: channel = sshclient.invoke_shell() # 如果shell在200s秒内无任何输出,则会中断 channel.settimeout(200) stdin = channel.makefile('wb') stdout = channel.makefile('rb') stderr = channel.makefile_stderr('rb') if self.srvid == "QS-CMSA-union": stdin.write(''' \n %s y exit ''' % command) else: stdin.write(''' %s exit ''' % command) output = "" errs = "" try: output = stdout.read() errs = stderr.read() except Exception, e: mylogger.error(e) if len(output) > 0: mylogger.info(output) if len(errs) > 0: mylogger.error(errs) if self.needcheck == "YES": for i in range(15): if self.check_service(sshclient): mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True time.sleep(1) else: mylogger.info("Check is passed, service start considered success.") mylogger.info("Common start %s [SUCCESS]..." % self.__str__()) self.status = "STARTED" self.release(sshclient) return True except Exception as e: mylogger.error(e) return False finally: mylogger.debug(self.get_separator("end"))
class Service(): def __init__(self, srvid, ip, username, password, nohupstart, start, nohupstop, stop, needcheck, check, status): # Service class definition # Attributes: # nohupstart: YES/NO # nohupstart: YES/NO # status: STARTED/STOPPED # start, stop, check: # Script or command to do start/stop/check which stored on destination hosts. # needcheck: Signal of if check needed self.srvid = srvid self.ip = ip self.username = username self.password = password self.nohupstart = nohupstart self.start = start self.nohupstop = nohupstop self.stop = stop self.needcheck = needcheck self.check = check self.status = status def __str__(self): """ :return a string which represent service object """ return "%s@%s" % (self.srvid, self.ip) def conn(self): """ Create ssh connection. 启动服务. Create ssh connection using parameters given to service instance,using module paramiko. 使用paramiko模块,通过service实例给定参数建立ssh连接。 Args: 参数: ip,username,password 指定ssh超时时间timeout 5s Returns: 返回值: Return a ssh connection . 返回一个ssh连接。 Raises: exceptions 发生异常时时记录异常信息 """ try: ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=self.ip, username=self.username, password=self.password, timeout=5) mylogger.debug("Connected to %s." % self.ip) return ssh except Exception as exceptions: mylogger.error(exceptions) def release(self, ssh): """ Disconnect a ssh connection. 释放连接. Release a ssh connection using close() 释放ssh连接 Args: 参数: sshclient Returns: no return values Raises: exceptions 发生异常时时记录异常信息 """ try: ssh.close() mylogger.debug("Disconnected from %s." % self.ip) except Exception as exceptions: mylogger.error(exceptions) finally: ssh.close() def get_separator(self, name): string = "----------------------" new_string = "" if name == "begin": new_string = string + "[Begin] " + self.__str__() + string elif name == "end": new_string = string + "[Ended] " + self.__str__() + string return new_string @fn_timer_logger def start_service(self): """ start service . 启动服务. Start service by running scripts like db2start.sh on server. 通过执行服务器上的启动脚本,如db2start.sh,启动服务。 Args: 参数: sshclient, nohupstart, start ssh连接,是否为nohup启动方式:YES/NO,启动脚本:/path/startscript.sh Returns: 返回值: Return boolean values.True while service start successfully,False while not. 返回布尔类型值:True代表服务启动成功,False反之。 Raises: exceptions 发生异常时时记录异常信息,返回False """ try: mylogger.debug(self.get_separator("begin")) sshclient = self.conn() command = self.start if self.nohupstart == "YES": mylogger.info("NOHUP Starting %s ..." % self.__str__()) mylogger.debug(command) channel = sshclient.get_transport().open_session() mylogger.debug(command) channel.exec_command(command) if self.needcheck == "YES": for i in range(15): if self.check_service(sshclient): mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True time.sleep(1) # if for loop ended without return ,means check failed. mylogger.info("NOHUP start %s [FAILED]!" % self.__str__()) self.release(sshclient) return False else: mylogger.info("Check is NO, service started success by default.") mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True else: """ Common way startup: simulate a shell, execute command, and exit shell return to python program """ mylogger.info("Common Starting %s ..." % self.__str__()) mylogger.debug(command) srvid = self.srvid if srvid.__contains__("DB2"): stdin, stdout, stderr = sshclient.exec_command(command) output = stdout.read() errs = stderr.read() else: channel = sshclient.invoke_shell() # 如果shell在200s秒内无任何输出,则会中断 channel.settimeout(200) stdin = channel.makefile('wb') stdout = channel.makefile('rb') stderr = channel.makefile_stderr('rb') if self.srvid == "QS-CMSA-union": stdin.write(''' \n %s y exit ''' % command) else: stdin.write(''' %s exit ''' % command) output = "" errs = "" try: output = stdout.read() errs = stderr.read() except Exception, e: mylogger.error(e) if len(output) > 0: mylogger.info(output) if len(errs) > 0: mylogger.error(errs) if self.needcheck == "YES": for i in range(15): if self.check_service(sshclient): mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__()) self.status = "STARTED" self.release(sshclient) return True time.sleep(1) else: mylogger.info("Check is passed, service start considered success.") mylogger.info("Common start %s [SUCCESS]..." % self.__str__()) self.status = "STARTED" self.release(sshclient) return True except Exception as e: mylogger.error(e) return False finally: mylogger.debug(self.get_separator("end")) @fn_timer_logger def stop_service(self): """ stop service . 停止服务. Stop service by running scripts like db2stop.sh on server. 通过执行服务器上的服务检查脚本,如db2stop.sh,停止服务。 Args: 参数: sshclient, nohupstop, stop Returns: 返回值: Return boolean values.True while service stop successfully,False while not. 返回布尔类型值:True代表服务停止成功,False反之。 Raises: exceptions 发生异常时时记录异常信息 """ try: command = self.stop if command == "": mylogger.info("Check is NULL! service stop considered success.") mylogger.info("Stop %s [SUCCESS]!" % self.__str__()) self.status = "STOPPED" return True mylogger.debug(self.get_separator("begin")) sshclient = self.conn() mylogger.info("Stopping %s ..." % self.__str__()) mylogger.debug(command) channel = sshclient.invoke_shell() # 如果shell在200s秒内无任何输出,则会中断 channel.settimeout(200) stdin = channel.makefile('wb') stdout = channel.makefile('rb') stderr = channel.makefile_stderr('rb') if self.srvid == "QS-CMSA-union": stdin.write(''' \n %s y exit ''' % command) else: stdin.write(''' %s exit ''' % command) output, errs = "", "" try: output = stdout.read() errs = stderr.read() except Exception, e: mylogger.error(e) mylogger.info(output) mylogger.error(errs) # 默认认为停服务成功 mylogger.info("Check is passed for stop, service stop considered success.") mylogger.info("Stop %s [SUCCESS]!" % self.__str__()) self.status = "STOPPED" self.release(sshclient) return True except Exception as e: mylogger.error(e)
def stop_service(self): """ stop service . 停止服务. Stop service by running scripts like db2stop.sh on server. 通过执行服务器上的服务检查脚本,如db2stop.sh,停止服务。 Args: 参数: sshclient, nohupstop, stop Returns: 返回值: Return boolean values.True while service stop successfully,False while not. 返回布尔类型值:True代表服务停止成功,False反之。 Raises: exceptions 发生异常时时记录异常信息 """ try: command = self.stop if command == "": mylogger.info("Check is NULL! service stop considered success.") mylogger.info("Stop %s [SUCCESS]!" % self.__str__()) self.status = "STOPPED" return True mylogger.debug(self.get_separator("begin")) sshclient = self.conn() mylogger.info("Stopping %s ..." % self.__str__()) mylogger.debug(command) channel = sshclient.invoke_shell() # 如果shell在200s秒内无任何输出,则会中断 channel.settimeout(200) stdin = channel.makefile('wb') stdout = channel.makefile('rb') stderr = channel.makefile_stderr('rb') if self.srvid == "QS-CMSA-union": stdin.write(''' \n %s y exit ''' % command) else: stdin.write(''' %s exit ''' % command) output, errs = "", "" try: output = stdout.read() errs = stderr.read() except Exception, e: mylogger.error(e) mylogger.info(output) mylogger.error(errs) # 默认认为停服务成功 mylogger.info("Check is passed for stop, service stop considered success.") mylogger.info("Stop %s [SUCCESS]!" % self.__str__()) self.status = "STOPPED" self.release(sshclient) return True
# Filename: fetchimg.py # Author: # sdkyoku # Created: 2014-09-08 18:18 __version__ = '1.1.0' from logger import mylogger as logger from urllib import unquote import sys try: from bs4 import BeautifulSoup except ImportError as ie: logger.error('%s', ie) sys.stdout.write('Cannot found module \"bs4\", please install \"beautifulsoup4\" with easy_install or pip.\n') sys.exit(77) import ConfigParser import sqlite3 import threading import urllib2 import socket try: import socks except ImportError as ie: logger.error('%s', ie) sys.stdout.write('Cannot found module \"socks\", please install \"socks\" with easy_install or pip.\n') sys.exit(77)