def download(self, dic, chunk_size=1024*4, progressBar=show_progress):
"""将取得的内容写入文件"""
try:
(resp, pos, size, total) = self.get_response(dic)
except TypeError as nt:
logger.error('%s', nt.reason)
bsf = pos
logger.debug('bsf: %d', bsf)
_filename = get_filename(dic['FILENAME'])
if bsf != 0:
fh = open(_filename, 'ab')
total = total+pos
logger.info('Resume mode: start from %d', pos)
elif bsf == 0:
fh = open(_filename, 'wb')
while True:
try:
buf = resp.read(chunk_size)
if not buf:
break
fh.write(buf)
bsf+=len(buf)
progressBar(total, bsf, dic['HASH'], self.queue.qsize())
except (urllib2.HTTPError, ssl.SSLError):
logger.error('\nTIMEOUT')
continue
fh.flush()
fh.close()
try:
sys.stdout.write('\n')
db_handle.record(dic)
except:
pass
def conn(self):
""" Create ssh connection.
启动服务.
Create ssh connection using parameters given to service instance,using module paramiko.
使用paramiko模块，通过service实例给定参数建立ssh连接。
Args:
参数：
ip,username,password
指定ssh超时时间timeout 5s
Returns:
返回值：
Return a ssh connection .
返回一个ssh连接。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=self.ip, username=self.username, password=self.password, timeout=5)
mylogger.debug("Connected to %s." % self.ip)
return ssh
except Exception as exceptions:
mylogger.error(exceptions)
def disconnect(self):
try:
conn.commit()
cur.close()
conn.close()
except sqlite3.DatabaseError, e:
if isinstance(e.args[0], sqlite3.DatabaseError):
logger.error('%s', e.args[0])
else:
raise
def get_args():
arg_length = sys.argv.__len__()
system, operation, service1, service2 = "", "", "", ""
if arg_length not in (3, 4, 5):
print_usage()
exit(1)
elif arg_length == 3:
# qs start | stop | check | list
system = sys.argv[1]
operation = sys.argv[2]
if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank",
"wailian",
"3par") or operation not in ("start", "stop",
"check", "list"):
print_usage()
exit(1)
elif arg_length == 4:
# qs start | stop | check service1
system = sys.argv[1]
operation = sys.argv[2]
service1 = sys.argv[3]
if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank",
"wailian", "3par") or operation not in ("start",
"stop",
"check"):
print_usage()
exit(1)
if not get_srv_ids(system).__contains__(service1):
print_usage()
exit(1)
elif arg_length == 5:
# qs start | stop | check service1 service2
system = sys.argv[1]
operation = sys.argv[2]
service1 = sys.argv[3]
service2 = sys.argv[4]
if system not in ("qs", "ndp", "ib", "zf", "zh", "hg", "ra", "sambank",
"wailian", "3par") or operation not in ("start",
"stop",
"check"):
print_usage()
exit(1)
idlist = get_srv_ids(system)
if not idlist.__contains__(service1) or not idlist.__contains__(
service2):
print_usage()
exit(1)
if idlist.index(service1) > idlist.index(service2):
print_usage()
mylogger.error(
"PARAMETER ERROR: service2 should be after service1 !!!")
exit(1)
return system, operation, service1, service2
def get_json(system):
try:
json_name = system + "_workflow.json"
if os.path.exists(json_name):
f = codecs.open(json_name, mode='r', encoding='utf-8')
service_list = json.load(f)
f.close()
return service_list
else:
mylogger.error("Json file not found.")
exit(1)
except Exception as exp:
mylogger.error(exp)
exit(1)
def print_usage():
command = ""
for arg in sys.argv:
command = command + " " + arg
mylogger.error("PARAMETER ERROR: %s " % command)
mylogger.error(
"COMMAND FORMAT: python auto.py [system] [operation] [service1] [service2]"
)
mylogger.error(
" -- system: qs | sambank | ib | ndp | zf | zh | hg | ra | wailian "
)
mylogger.error(" -- operation: start | stop | check | list")
mylogger.error(
" -- service1/service2: Get serviceid using 'list' argument"
)
def dostart(srvid, ip, username, password, nohupstart, start, nohupstop, stop,
needcheck, check, status):
service = Service(srvid, ip, username, password, nohupstart, start,
nohupstop, stop, needcheck, check, status)
if service.start_service():
mylogger.info(
"[------------ %s start [success]. Going to next service >>>>>>>>>>]"
% service.__str__())
return True
else:
mylogger.error(
"[------------ %s start [failed]. Workflow terminated! XXXXXXXXXX]"
% service.__str__())
return False
def get_srv_ids(system):
try:
json_name = system + "_workflow.json"
if os.path.exists(json_name):
f = codecs.open(json_name, mode='r', encoding='utf-8')
service_list = json.load(f)
f.close()
srvids = []
for service in service_list:
srvids.append(service["srvid"])
return srvids
else:
mylogger.error("Json file not found.")
exit(1)
except Exception as exp:
mylogger.error(exp)
exit(1)
def get_response(self, dic):
pos = get_file_size(get_filename(dic['FILENAME']))
logger.debug('Filename: %s', get_filename(dic['FILENAME']))
logger.debug('Position: %d', pos)
try:
req = urllib2.Request(dic['URL'])
if pos != 0:
req.headers['Range'] = 'bytes=%s-' % pos
resp = urllib2.urlopen(req, timeout=30)
totalsize = resp.info().getheader('Content-Length').strip()
totalsize = int(totalsize)
size = totalsize-pos
return (resp, pos, size, totalsize)
except (urllib2.HTTPError, ssl.SSLError, socket.timeout):
# handle timeout, nothing here
logger.error('%s', 'time out.')
except urllib2.URLError as e:
logger.error('%s', e.reason)
def handle_link(self, link):
# 因为有些网址里含有CJK字符，所以在这个里边需要进行一下转码
link = unicode(link)
req = urllib2.Request(link, headers={'User-Agent': UA})
resp = urllib2.urlopen(req)
cont = resp.read()
soup = BeautifulSoup(cont)
try:
resu = soup.find_all('a', class_='original-file-unchanged')
url = resu[0].get('href')
except IndexError as ie:
logger.error('Lossless version not found, download changed version.')
resu = soup.find_all('a', class_='original-file-changed')
url = resu[0].get('href')
# result = PATTERN.match(cont).group(1)
# logger.info('Matched: %s', result)
logger.info('Matched: %s', url)
return url
def release(self, ssh):
""" Disconnect a ssh connection.
释放连接.
Release a ssh connection using close()
释放ssh连接
Args:
参数：
sshclient
Returns:
no return values
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh.close()
mylogger.debug("Disconnected from %s." % self.ip)
except Exception as exceptions:
mylogger.error(exceptions)
finally:
ssh.close()
def check_service(self, sshclient):
""" Check service status.
使用传入连接，检查服务状态，以解决重复创建连接问题.
Checking service status by running scripts like db2check.sh on server.
通过执行服务器上的服务检查脚本，如db2check.sh，确定服务是否活动。
Args:
参数：
check,status
检查脚本，状态
Returns:
返回值：
Return boolean values.True while service running,False while not.
返回布尔类型值：True代表服务已启动，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
command = self.check
if command == "":
mylogger.error("CHECK ERROR:check command is null.")
return False
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
mylogger.debug(command)
mylogger.info(output)
# only way to return success is get keyword "_SUCCESS_"
if len(output) > 0:
if output.__contains__("_SUCCESS_"):
mylogger.info("CHECK %s [SUCCESS]!" % self.__str__())
return True
else:
mylogger.info("CHECK %s [FAILED]!" % self.__str__())
return False
if len(errs) > 0:
mylogger.error("CHECK ERROR:command error happened.")
mylogger.error(errs)
return False
except Exception as e:
mylogger.error(e)
def url_eater(self, queue):
"""
接受用户输入，将检测结果合法的元素放入队列中
"""
try:
while 1:
sys.stdout.write('Enter url: ')
tmp_url = sys.stdin.readline().strip('\n')
tmp_time = time.ctime()
if tmp_url == 'q':
logger.debug('Thread pool size: %d', len(self.thread_list))
if len(self.thread_list) == 0:
db_handle.disconnect()
sys.exit(0)
else:
logger.error('Thread Fetcher is working! Please retry until it done.')
time.sleep(3)
continue
if (tmp_url.find('http', 0) == -1 and tmp_url.find('://', 8) != -1):
logger.error('Invalid URL: Are you forget the head of http or enter multiple url here?')
break
if (tmp_url.split('.')[-1] != 'png' and tmp_url.split('.')[-1] != 'jpg'):
tmp = self.handle_link(tmp_url)
tmp_url = tmp
# logger.error('Invalid file extension: %s.', tmp_url.split(r'/')[-1])
# continue
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
logger.debug('WAIT@EATER.')
if len(self.thread_list) == 0:
fetcher = Fetcher('Link Downloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
tmp_dict = {
'URL': tmp_url,
'FILENAME': unquote(tmp_url.split(r'/')[-1]),
'HASH': tmp_url.split(r'/')[-2],
'APTIME': tmp_time,
'FLG': False
}
logger.debug('\nURL: %s\nFILENAME: %s\nHASH: %s\nAPTIME: %s', tmp_dict['URL'], tmp_dict['FILENAME'], tmp_dict['HASH'], tmp_dict['APTIME'])
completed = db_handle.query(tmp_dict['HASH'])
if completed == 0:
logger.info('%s already downloaded.', tmp_dict['HASH'])
continue
try:
logger.debug('ACQUIRED BY EATER')
queue.put(tmp_dict)
except:
pass
finally:
logger.info('%s ADDED, QUEUE SIZE %s', tmp_dict['HASH'], self.queue.qsize())
if len(self.thread_list) == 0:
fetcher = Fetcher('URL doenloader', self.queue, thread_list)
fetcher.setDaemon(True)
self.thread_list.append(fetcher)
logger.debug('Thread list contains: %s', thread_list)
fetcher.start()
else:
logger.debug('Pool of fetcher is full.')
except KeyboardInterrupt:
logger.debug('User aborted, but I will record recent downloads.')
db_handle.disconnect()
sys.exit(99)
finally:
logger.debug('Thread list contains: %s', thread_list)
time.sleep(1)
def main():
# program run only by devops
if getpass.getuser() != "devops":
mylogger.error("This program should only be run by devops!")
exit(99)
command = ""
for arg in sys.argv:
command = command + " " + arg
mylogger.info(get_separator(command, "line"))
mylogger.info(get_separator(command, "begin"))
mylogger.info(get_separator(command, "line"))
try:
system, operation, service1, service2 = get_args()
service_list = get_json(system)
if service1 == "" and service2 == "":
if operation == "start":
for service in service_list:
if dostart(**service):
pass
else:
break
elif operation == "stop":
service_list.reverse()
for service in service_list:
if dostop(**service):
pass
else:
break
elif operation == "check":
for service in service_list:
docheck(**service)
elif operation == "list":
for service in service_list:
mylogger.info("[ " + service["srvid"] + " ]" + ": " +
service["start"])
elif service2 == "":
# start one service by id
if operation == "start":
for service in service_list:
if service["srvid"] == service1:
dostart(**service)
elif operation == "stop":
for service in service_list:
if service["srvid"] == service1:
dostop(**service)
elif operation == "check":
for service in service_list:
if service["srvid"] == service1:
docheck(**service)
else:
idlist = get_srv_ids(system)
i = idlist.index(service1)
j = idlist.index(service2) + 1
service_list1 = service_list.__getslice__(i, j)
if operation == "start":
for service in service_list1:
if dostart(**service):
pass
else:
break
elif operation == "stop":
service_list1.reverse()
for service in service_list1:
if dostop(**service):
pass
else:
break
elif operation == "check":
for service in service_list1:
docheck(**service)
finally:
mylogger.info(get_separator(command, "line"))
mylogger.info(get_separator(command, "end"))
mylogger.info(get_separator(command, "line"))
def start_service(self):
""" start service .
启动服务.
Start service by running scripts like db2start.sh on server.
通过执行服务器上的启动脚本，如db2start.sh，启动服务。
Args:
参数：
sshclient, nohupstart, start
ssh连接，是否为nohup启动方式:YES/NO，启动脚本:/path/startscript.sh
Returns:
返回值：
Return boolean values.True while service start successfully,False while not.
返回布尔类型值：True代表服务启动成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
command = self.start
if self.nohupstart == "YES":
mylogger.info("NOHUP Starting %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.get_transport().open_session()
mylogger.debug(command)
channel.exec_command(command)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
# if for loop ended without return ,means check failed.
mylogger.info("NOHUP start %s [FAILED]!" % self.__str__())
self.release(sshclient)
return False
else:
mylogger.info("Check is NO, service started success by default.")
mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
else:
"""
Common way startup:
simulate a shell, execute command, and exit shell return to python program
"""
mylogger.info("Common Starting %s ..." % self.__str__())
mylogger.debug(command)
srvid = self.srvid
if srvid.__contains__("DB2"):
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
else:
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output = ""
errs = ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
if len(output) > 0:
mylogger.info(output)
if len(errs) > 0:
mylogger.error(errs)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
else:
mylogger.info("Check is passed, service start considered success.")
mylogger.info("Common start %s [SUCCESS]..." % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
return False
finally:
mylogger.debug(self.get_separator("end"))
class Service():
def __init__(self, srvid, ip, username, password, nohupstart, start, nohupstop, stop, needcheck, check, status):
# Service class definition
# Attributes:
# nohupstart: YES/NO
# nohupstart: YES/NO
# status: STARTED/STOPPED
# start, stop, check:
# Script or command to do start/stop/check which stored on destination hosts.
# needcheck: Signal of if check needed
self.srvid = srvid
self.ip = ip
self.username = username
self.password = password
self.nohupstart = nohupstart
self.start = start
self.nohupstop = nohupstop
self.stop = stop
self.needcheck = needcheck
self.check = check
self.status = status
def __str__(self):
"""
:return a string which represent service object
"""
return "%s@%s" % (self.srvid, self.ip)
def conn(self):
""" Create ssh connection.
启动服务.
Create ssh connection using parameters given to service instance,using module paramiko.
使用paramiko模块，通过service实例给定参数建立ssh连接。
Args:
参数：
ip,username,password
指定ssh超时时间timeout 5s
Returns:
返回值：
Return a ssh connection .
返回一个ssh连接。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=self.ip, username=self.username, password=self.password, timeout=5)
mylogger.debug("Connected to %s." % self.ip)
return ssh
except Exception as exceptions:
mylogger.error(exceptions)
def release(self, ssh):
""" Disconnect a ssh connection.
释放连接.
Release a ssh connection using close()
释放ssh连接
Args:
参数：
sshclient
Returns:
no return values
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
ssh.close()
mylogger.debug("Disconnected from %s." % self.ip)
except Exception as exceptions:
mylogger.error(exceptions)
finally:
ssh.close()
def get_separator(self, name):
string = "----------------------"
new_string = ""
if name == "begin":
new_string = string + "[Begin] " + self.__str__() + string
elif name == "end":
new_string = string + "[Ended] " + self.__str__() + string
return new_string
@fn_timer_logger
def start_service(self):
""" start service .
启动服务.
Start service by running scripts like db2start.sh on server.
通过执行服务器上的启动脚本，如db2start.sh，启动服务。
Args:
参数：
sshclient, nohupstart, start
ssh连接，是否为nohup启动方式:YES/NO，启动脚本:/path/startscript.sh
Returns:
返回值：
Return boolean values.True while service start successfully,False while not.
返回布尔类型值：True代表服务启动成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息，返回False
"""
try:
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
command = self.start
if self.nohupstart == "YES":
mylogger.info("NOHUP Starting %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.get_transport().open_session()
mylogger.debug(command)
channel.exec_command(command)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
# if for loop ended without return ,means check failed.
mylogger.info("NOHUP start %s [FAILED]!" % self.__str__())
self.release(sshclient)
return False
else:
mylogger.info("Check is NO, service started success by default.")
mylogger.info("NOHUP start%s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
else:
"""
Common way startup:
simulate a shell, execute command, and exit shell return to python program
"""
mylogger.info("Common Starting %s ..." % self.__str__())
mylogger.debug(command)
srvid = self.srvid
if srvid.__contains__("DB2"):
stdin, stdout, stderr = sshclient.exec_command(command)
output = stdout.read()
errs = stderr.read()
else:
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output = ""
errs = ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
if len(output) > 0:
mylogger.info(output)
if len(errs) > 0:
mylogger.error(errs)
if self.needcheck == "YES":
for i in range(15):
if self.check_service(sshclient):
mylogger.info("NOHUP start %s [SUCCESS]!" % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
time.sleep(1)
else:
mylogger.info("Check is passed, service start considered success.")
mylogger.info("Common start %s [SUCCESS]..." % self.__str__())
self.status = "STARTED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
return False
finally:
mylogger.debug(self.get_separator("end"))
@fn_timer_logger
def stop_service(self):
""" stop service .
停止服务.
Stop service by running scripts like db2stop.sh on server.
通过执行服务器上的服务检查脚本，如db2stop.sh，停止服务。
Args:
参数：
sshclient, nohupstop, stop
Returns:
返回值：
Return boolean values.True while service stop successfully,False while not.
返回布尔类型值：True代表服务停止成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
command = self.stop
if command == "":
mylogger.info("Check is NULL! service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
return True
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
mylogger.info("Stopping %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output, errs = "", ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
mylogger.info(output)
mylogger.error(errs)
# 默认认为停服务成功
mylogger.info("Check is passed for stop, service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
self.release(sshclient)
return True
except Exception as e:
mylogger.error(e)
def stop_service(self):
""" stop service .
停止服务.
Stop service by running scripts like db2stop.sh on server.
通过执行服务器上的服务检查脚本，如db2stop.sh，停止服务。
Args:
参数：
sshclient, nohupstop, stop
Returns:
返回值：
Return boolean values.True while service stop successfully,False while not.
返回布尔类型值：True代表服务停止成功，False反之。
Raises:
exceptions
发生异常时时记录异常信息
"""
try:
command = self.stop
if command == "":
mylogger.info("Check is NULL! service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
return True
mylogger.debug(self.get_separator("begin"))
sshclient = self.conn()
mylogger.info("Stopping %s ..." % self.__str__())
mylogger.debug(command)
channel = sshclient.invoke_shell()
# 如果shell在200s秒内无任何输出，则会中断
channel.settimeout(200)
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
if self.srvid == "QS-CMSA-union":
stdin.write('''
\n
%s
y
exit
''' % command)
else:
stdin.write('''
%s
exit
''' % command)
output, errs = "", ""
try:
output = stdout.read()
errs = stderr.read()
except Exception, e:
mylogger.error(e)
mylogger.info(output)
mylogger.error(errs)
# 默认认为停服务成功
mylogger.info("Check is passed for stop, service stop considered success.")
mylogger.info("Stop %s [SUCCESS]!" % self.__str__())
self.status = "STOPPED"
self.release(sshclient)
return True
# Filename: fetchimg.py
# Author:
# sdkyoku
# Created: 2014-09-08 18:18
__version__ = '1.1.0'
from logger import mylogger as logger
from urllib import unquote
import sys
try:
from bs4 import BeautifulSoup
except ImportError as ie:
logger.error('%s', ie)
sys.stdout.write('Cannot found module \"bs4\", please install \"beautifulsoup4\" with easy_install or pip.\n')
sys.exit(77)
import ConfigParser
import sqlite3
import threading
import urllib2
import socket
try:
import socks
except ImportError as ie:
logger.error('%s', ie)
sys.stdout.write('Cannot found module \"socks\", please install \"socks\" with easy_install or pip.\n')
sys.exit(77)