def __child_watch_callback (self, pid, code):
# Kill the engine on any signal but 'Resource temporarily unavailable'
if code != errno.EWOULDBLOCK:
if type(code) == str:
log.error(code+"\n", self.defname)
else: log.error(os.strerror(code)+"\n", self.defname)
self.emit("died")
self.gentleKill()
def _disconnect(self):
"""
Closes session
"""
try:
self.ssh.close()
except Exception as e:
log.error(e)
def __child_watch_callback(self, pid, code):
# Kill the engine on any signal but 'Resource temporarily unavailable'
if code != errno.EWOULDBLOCK:
if type(code) == str:
log.error(code + "\n", self.defname)
else:
log.error(os.strerror(code) + "\n", self.defname)
self.emit("died")
self.gentleKill()
def acquireIp():
aUrl = getUrl()
log.info('获取ip地址:{}'.format(aUrl))
try:
reponse = requests.get(aUrl, headers=header, timeout=5)
if reponse.status_code == 200:
parseHtml(reponse.text)
except:
# traceback.print_exc()
log.error('请求ip异常:{}'.format(aUrl))
def read_json(cfile: str) -> list:
"""
Opens JSON conf file and returns its contents.
"""
import json
try:
with open(cfile, 'r') as json_file:
return json.load(json_file)
except Exception as e:
log.error(e)
def updata():
log.info('更新线程启动！！！')
while (True):
try:
acquire(1)
time.sleep(6)
except:
traceback.print_exc()
log.error("更新时有异常。。。。")
time.sleep(2)
def write(self, data):
if self.channelsClosed:
log.warn("Chan closed for %r" % data, self.defname)
return
log.log(data, self.defname)
self.inChannel.write(data)
if data.endswith("\n"):
try:
self.inChannel.flush()
except gobject.GError, e:
log.error(str(e) + ". Last line wasn't sent.\n", self.defname)
def checkIpMain():
while True:
try:
log.info('测试线程执行！！！')
testIp()
deleteIp()
time.sleep(6)
except:
traceback.print_exc()
log.error("测试时有异常。。。。")
time.sleep(2)
def read_yaml(cfile: str) -> list:
"""
Opens YAML conf file and returns its contents.
"""
import yaml
try:
with open(cfile, 'r') as yaml_file:
conf = yaml.load(yaml_file)
return [each for each in conf.items()]
except Exception as e:
log.error(e)
def write (self, data):
if self.channelsClosed:
log.warn("Chan closed for %r" % data, self.defname)
return
log.log(data, self.defname)
self.inChannel.write(data)
if data.endswith("\n"):
try:
self.inChannel.flush()
except gobject.GError, e:
log.error(str(e)+". Last line wasn't sent.\n", self.defname)
def parse_map_value(puzzle):
value = [0] * (env.size * env.size)
for line in puzzle:
for num in line:
if num >= env.size * env.size:
log.error("parsing map. value find to high : " + str(num))
sys.exit(1)
if value[num] == 1:
log.error("parsing map. value find multiple time : " + str(num))
sys.exit(1)
value[num] = 1
return
def instruct(self, command: str):
"""
Sends instruction to a device
"""
self._connect()
try:
stdin, stdout, stderr = self.ssh.exec_command(command)
out = stdout.read()
self._disconnect()
return out
except Exception as e:
log.error(e)
def _connect(self):
"""
Establishes ssh connection with given session parameters
"""
try:
self.ssh.connect(hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
allow_agent=False,
look_for_keys=False)
except Exception as e:
log.error(e)
def read_ini(cfile: str) -> list:
"""
Opens INI conf file and returns its contents.
"""
from configparser import ConfigParser
config = ConfigParser()
out = []
try:
config.read(cfile)
sections = config.sections()
for section in sections:
items = config.items(section)
out.append({section: {item[0]: item[1] for item in items}})
return out
except Exception as e:
log.error(e)
def read_xml(cfile: str) -> list:
"""
Opens XML conf file and returns its contents.
"""
import xml.etree.ElementTree as xmlparser
out = []
try:
tree = xmlparser.parse(cfile)
root = tree.getroot()
for device in root.findall('device'):
out.append(
{device.attrib['name']:
{'ip': device.find('ip').text,
'user': device.find('user').text,
'password': device.find('password').text}})
return out
except Exception as e:
log.error(e)
def __io_cb (self, channel, condition, isstderr):
while True:
try:
line = channel.next()#readline()
except StopIteration:
self._wait4exit()
self.__child_watch_callback(*self.subprocExitCode)
break
if not line:
return True
if isstderr:
log.error(line, self.defname)
else:
for word in self.warnwords:
if word in line:
log.warn(line, self.defname)
break
else: log.debug(line, self.defname)
self.linePublisher.put(line)
def __io_cb(self, channel, condition, isstderr):
while True:
try:
line = channel.next() #readline()
except StopIteration:
self._wait4exit()
self.__child_watch_callback(*self.subprocExitCode)
break
if not line:
return True
if isstderr:
log.error(line, self.defname)
else:
for word in self.warnwords:
if word in line:
log.warn(line, self.defname)
break
else:
log.debug(line, self.defname)
self.linePublisher.put(line)
def parse_line(line):
if (len(re.findall(r"([^\d\s]+)", line)) > 0):
log.error("parsing line (bad character found) " + line)
sys.exit(1)
m = re.findall(r"(\s*-?\d+)", line)
if (parse_line.first is True):
parse_line.first = False
if len(m) != 1:
log.error("parsing map. First Line must be the map size")
sys.exit(1)
env.size = int(m[0])
if not env.is_valid_size(env.size):
log.error("Invalid map size: must be higher or equal than 2")
sys.exit(1)
else:
if len(m) < env.size:
log.error("parsing map. not enought numbers at line : " + line)
sys.exit(1)
elif len(m) > env.size:
log.error("parsing map. too many numbers at line : " + line)
sys.exit(1)
return [ to_int(num) for num in m ]
def parse_map(map):
first = True
puzzle = []
for line in map:
line_util = line.split('#')[0]
if line_util == None or line_util == "":
continue
if first is not True:
puzzle.append(parse_line(line_util))
if (len(puzzle) > env.size):
log.error("parsing map. Too many lines")
sys.exit(1)
else:
first = False
parse_line(line_util)
if (len(puzzle) < env.size):
log.error("parsing map. Not Enought Lines")
sys.exit(1)
parse_map_value(puzzle)
return Puzzle(puzzle)
def crawl(self, root_url: str, depth=0):
self._url_list.append(root_url)
current_idx = 0
while current_idx < len(self._url_list):
url = self._url_list[current_idx]
print(80*"-")
log.info(f"Processing {url}")
current_idx += 1
if is_url_filtered(url):
log.info("URL is filtered, skipping")
continue
if len(url) >= self._max_url_length:
log.info(f"URL is too long (max_length={self._max_url_length}), skipping")
continue
try:
if not self.is_html(url):
log.info("URL is not HTML, skipping")
continue
req = Request(url, headers=self._header)
response = urlopen(req, timeout=3)
content = response.read().decode(errors='ignore')
except Exception as e:
log.error(e)
log.info("An error occurred while opening URL, skipping")
continue
# detect if url is an entirely new domain and reset counter
if self.get_domain(url) != self._previous_domain:
self._current_on_site = 0
else:
self._current_on_site += 1
self._previous_domain = self.get_domain(url)
# get title and check whether it's latin
title = self.get_title(content)
if title:
title = title.strip()
if self.is_latin(title):
keywords = get_keywords(content)
self._indexer.save(url, title, keywords)
else:
log.info(f"Skipping because: Title not latin ('{title}')")
continue
# extract links from html
soup = BeautifulSoup(content, features="lxml")
to_crawl = []
cnt = 0
for link in soup.findAll('a'):
l = link.get('href')
if l and (l.startswith("http") or l[0] == '/'):
if l[0] == '/':
l = urljoin(url, l)
# discard too many links on same domain to prevent cycles
if self._current_on_site <= self._max_stay_on_site:
if not self._indexer.url_already_indexed(l) and l not in self._url_list:
self._url_list.append(l)
cnt += 1
# but make sure to append 'foreign' URLs in every case
if self.get_domain(url) != self.get_domain(l):
self._url_list.append(l)
cnt += 1
if cnt >= self._max_new_urls_per_page:
break
log.info(f"URLs found: {len(self._url_list)} ({cnt} new)")
# check whether to clean URL list so it doesn't get too big
if len(self._url_list) >= self._max_urls_in_list:
len_before = len(self._url_list)
self.purge_url_list(self.get_domain(url))
len_after = len(self._url_list)
log.info(f"Purged URL list (removed {len_before - len_after} entries)")
current_idx = 0
def insert_mongo(self, ip):
mydict = {"_id": ip, "ip": ip, "isValid": 1, "count": 0, "time": time.time()}
try:
x = self.account.insert_one(mydict)
except:
log.error("重复")
from threading import Lock
from gobject import GObject, SIGNAL_RUN_FIRST, TYPE_NONE
from Log import log
try:
import pygst
pygst.require('0.10')
import gst
except ImportError, e:
log.error("Unable to import gstreamer. All sound will be mute.\n%s" % e)
class Player(GObject):
__gsignals__ = {
'end': (SIGNAL_RUN_FIRST, TYPE_NONE, ()),
'error': (SIGNAL_RUN_FIRST, TYPE_NONE, (object, ))
}
def checkSound(self):
self.emit("error", None)
def play(self, uri):
pass
else:
class Player(GObject):
__gsignals__ = {
'end': (SIGNAL_RUN_FIRST, TYPE_NONE, ()),
'error': (SIGNAL_RUN_FIRST, TYPE_NONE, (object, ))
}
from threading import Lock
from gobject import GObject, SIGNAL_RUN_FIRST, TYPE_NONE
from Log import log
try:
import pygst
pygst.require('0.10')
import gst
except ImportError, e:
log.error("Unable to import gstreamer. All sound will be mute.\n%s" % e)
class Player (GObject):
__gsignals__ = {
'end': (SIGNAL_RUN_FIRST, TYPE_NONE, ()),
'error': (SIGNAL_RUN_FIRST, TYPE_NONE, (object,))
}
def checkSound(self):
#self.emit("error", None)
def play(self, uri):
pass
else:
class Player (GObject):
__gsignals__ = {
'end': (SIGNAL_RUN_FIRST, TYPE_NONE, ()),
'error': (SIGNAL_RUN_FIRST, TYPE_NONE, (object,))
}
def __init__(self):
GObject.__init__(self)
self.player = gst.element_factory_make("playbin")