def scann(self, thread):
for path in os.listdir(self.dir):
file_type = path.split('.')[-1]
print(file_type)
if file_type == 'jpg' or file_type == 'png':
utils.scan(self.dir + '/' + path)
self.text += ('成功渲染：' + self.dir + '/' + path)
self.text += '\n'
thread.trigger.emit()
def feature(LR_dir, HR_dir, npoints=1000):
LR_paths = scan(LR_dir)
HR_paths = scan(HR_dir)
if not LR_paths.count() == HR_paths.count():
raise Exception("LR and HR don't match")
LR_arrs = pd.Series(LR_paths.map(read_img))
HR_arrs = pd.Series(HR_paths.map(read_img))
LR_HR_arrs = pd.DataFrame(zip(LR_arrs, HR_arrs))
###
def feature_helper(LR_array3d, HR_array3d, npoints):
#print('new_image') # debug
np.random.seed(2019)
xs = np.random.randint(LR_array3d.shape[0], size=npoints)
ys = np.random.randint(LR_array3d.shape[1], size=npoints)
coordinates = pd.DataFrame({'x': xs, 'y': ys})
feat_mat = coordinates.apply(get_feature_helper,
LR_array3d=LR_array3d,
axis=1)
feat_mat = np.dstack(feat_mat).transpose(2, 0, 1)
lab_mat = coordinates.apply(get_label_helper,
LR_array3d=LR_array3d,
HR_array3d=HR_array3d,
axis=1)
lab_mat = np.dstack(lab_mat).transpose(2, 0, 1)
return pd.Series({'feature': feat_mat, 'label': lab_mat})
###
feat_lab = np.array(
LR_HR_arrs.apply(lambda x: feature_helper(*x, npoints), axis=1))
feat_mat = np.vstack(feat_lab[:, 0])
label_mat = np.vstack(feat_lab[:, 1])
return feat_mat, label_mat
def run(self):
while True:
status = utils.status(self._user, self._pass)
if status is not None:
mines = status['mines']
for mine in mines:
# print(mine)
if mine[1] + ' ' + mine[2] not in self.mines:
# print(mine[1]+' '+mine[2])
mine_mutex.acquire()
self._mines[mine[1] + ' ' + mine[2]] = (time.time() -
30, mine[0])
mine_mutex.release()
else:
mine_mutex.acquire()
self._mines[mine[1] + ' ' +
mine[2]] = (self._mines[mine[1] + ' ' +
mine[2]], mine[0])
mine_mutex.release()
wormholes = status['wormholes']
for wormhole in wormholes:
if wormhole[0] + ' ' + wormhole[1] not in self.wormholes:
wh_mutex.acquire()
self._wormholes[wormhole[0] + ' ' +
wormhole[1]] = wormhole
wh_mutex.release()
scanned = utils.scan_parser(
utils.scan(self._user, self._pass, status, self._config))
if scanned is not None:
mines = scanned['mines']
for mine in mines:
# print(mine)
if mine[1] + ' ' + mine[2] not in self.mines:
# print(mine[1]+' '+mine[2])
mine_mutex.acquire()
self._mines[mine[1] + ' ' +
mine[2]] = (time.time() - 30, mine[0])
mine_mutex.release()
else:
mine_mutex.acquire()
self._mines[mine[1] + ' ' +
mine[2]] = (self._mines[mine[1] + ' ' +
mine[2]][0],
mine[0])
mine_mutex.release()
wormholes = scanned['wormholes']
for wormhole in wormholes:
if wormhole[0] + ' ' + wormhole[
1] not in self.wormholes:
wh_mutex.acquire()
self._wormholes[wormhole[0] + ' ' +
wormhole[1]] = wormhole
wh_mutex.release()
time.sleep(.1)
if self._stop_event.is_set():
return
def plot_scan(X, y, function, params):
listform = params.items()
train, test = scan(X, y, function, params)
xvals = listform[0][1]
plt.plot(numpy.log(xvals), train, '--', label="Training")
plt.plot(numpy.log(xvals), test, label="Test")
plt.legend(loc="best")
plt.xlabel("log(%s)" % listform[0][0])
plt.ylabel("MAE (eV)")
plt.title("Optimization of %s" % listform[0][0])
plt.show()
def scan(domain, options):
logging.debug("[%s][pshtt]" % domain)
# cache output from pshtt
cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")
force = options.get("force", False)
data = None
if (force is False) and (os.path.exists(cache_pshtt)):
logging.debug("\tCached.")
raw = open(cache_pshtt).read()
data = json.loads(raw)
if (data.__class__ is dict) and data.get('invalid'):
return None
else:
logging.debug("\t %s %s" % (command, domain))
raw = utils.scan([
command, domain, '--json', '--user-agent',
'\"%s\"' % user_agent, '--timeout',
str(timeout), '--preload-cache', preload_cache
])
if not raw:
utils.write(utils.invalid({}), cache_pshtt)
logging.warn("\tBad news scanning, sorry!")
return None
data = json.loads(raw)
utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))
# pshtt scanner uses JSON arrays, even for single items
data = data[0]
row = []
for field in headers:
value = data[field]
# TODO: Fix this upstream
if (field != "HSTS Header") and (field != "HSTS Max Age") and (
field != "Redirect To"):
if value is None:
value = False
row.append(value)
yield row
def plot_scan_2d(X, y, function, params):
listform = params.items()
train, test = scan(X, y, function, params)
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(test, interpolation='nearest')
fig.colorbar(cax)
ax.xaxis.set_ticks(numpy.arange(0,len(listform[0][1])))
ax.set_xticklabels(listform[0][1])
ax.yaxis.set_ticks(numpy.arange(0,len(listform[1][1])))
ax.set_yticklabels(listform[1][1])
plt.xlabel(listform[0][0])
plt.ylabel(listform[1][0])
plt.show()
return train, test
def super_resolution(LR_dir, model_list, out_dir):
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
LR_paths = scan(LR_dir)
LR_leaves = LR_paths.map(path_leaf)
out_paths = LR_leaves.map(lambda x: os.path.join(out_dir, x))
###
def super_resolution_(LR_path, model_list, out_dir):
LR_arr3d = read_img(LR_path)
nrow = LR_arr3d.shape[0]
ncol = LR_arr3d.shape[1]
xs = np.repeat(list(range(nrow)), ncol)
ys = np.array(list(range(ncol)) * nrow)
coordinates = pd.DataFrame({'x': xs, 'y': ys})
feat_mat = coordinates.apply(get_feature_helper,
LR_array3d=LR_arr3d,
axis=1)
feat_mat = np.dstack(feat_mat).transpose(
2, 0, 1) # shape = (nrow * ncol, 8, 3)
pred_mat = np.empty((nrow * ncol, 4, 3))
for i in range(12):
pred_mat[:, i % 4,
i // 4] = model_list[i].predict(feat_mat[:, :, i // 4])
HR_array3d = reshape_helper(nrow, ncol, pred_mat)
LR_arr3d_resize = np.apply_along_axis(np.repeat, 1, LR_arr3d, 2)
LR_arr3d_resize = np.apply_along_axis(np.repeat, 0, LR_arr3d_resize, 2)
HR_array3d = HR_array3d + LR_arr3d_resize
HR_array3d = HR_array3d.astype('uint8')
HR_img = Image.fromarray(HR_array3d)
HR_img.save(out_dir)
###
HR_LR_paths = pd.DataFrame({'path': out_paths, 'LR_paths': LR_paths})
HR_LR_paths.apply(lambda x: super_resolution_(x[1], model_list, x[0]),
axis=1)
def test_one_port_close(self):
"""Test start port is 50000 and the port is close"""
dictionary = {}
listen = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.setdefaulttimeout(1)
result = listen.connect_ex(('127.0.0.1', 50000))
if result == 0:
key = 50000
value = True
dictionary[key] = value
else:
key = 50000
value = False
dictionary[key] = value
listen.close()
self.assertEqual(dictionary, utils.scan('127.0.0.1', 50000, 50000))
def test_all_port_close(self):
"""Test start port is 50010 and end port is 50020, and all ports close"""
dictionary = {}
for port in range(50000, 50020 + 1):
listen = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.setdefaulttimeout(1)
result = listen.connect_ex(('127.0.0.1', port))
if result == 0:
key = port
value = True
dictionary[key] = value
else:
key = port
value = False
dictionary[key] = value
listen.close()
self.assertEqual(dictionary, utils.scan('127.0.0.1', 50000, 50020))
def print_summary(history, hierarchy):
"""Split transaction summary per month and category."""
def mkstring(value):
return "%.2f (%d)" % (value[0], value[1])
def print_tree(node, indent="\t", level=0):
print(mkstring(node[0]))
if len(node) > 1:
for (label, branch) in sorted(node[1].items()):
print(indent * level + str(label), end=": ")
print_tree(branch, indent, level + 1)
tree = split(history, hierarchy)
statistics = lambda ts: [sum(t.amount for t in ts), len(ts)]
summary = scan(tree, statistics, lambda xs, ys: [x + y for (x, y) in zip(xs, ys)])
print("total", end=": ")
print_tree(summary)
def get_port_status(conn):
"""
This method used to use arguments to implement scan
funciton and send the dictionary to relative partner.
"""
if len(sys.argv) < 3 or len(sys.argv) > 4:
print('Please check your ip address, start_port or' \
'end_port is inputted correcttly or not')
sys.exit()
ip_address = sys.argv[1]
start_port = sys.argv[2]
end_port = ''
if len(sys.argv) == 3:
end_port = sys.argv[2]
else:
end_port = sys.argv[3]
conn.send(utils.scan(ip_address, start_port, end_port))
conn.close()
def test_port_open_and_close(self):
"""Test start port is 50000 and end port is 50020"""
dictionary = {}
for port in range(50000, 50020 + 1):
listen = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen.bind(('127.0.0.1', 50018))
listen.listen(1)
socket.setdefaulttimeout(1)
result = listen.connect_ex(('127.0.0.1', port))
if result == 0:
key = port
value = True
dictionary[key] = value
else:
key = port
value = False
dictionary[key] = value
listen.close()
self.assertEqual(dictionary, utils.scan('127.0.0.1', 50000, 50020))
def catchHomePage(uin, uout):
module_names = [
"HomePage",
"HouseCounts",
"LoupanLink", # 城市链接 2
"LoupanHomepage", # 首页 3
"XQPageLouPan", # 详情页 4
"PhotoLouPanStageOne", # 相册页 - 相册类型与总数 5
"PhotoLouPanStageTwo", # 相册页 - 页面类型为1，Ajax请求相册页面 6
"PhotoLouPanStageThree", # 相册页 - 相片下载 7
]
module_name = module_names[7]
threads = []
# 模块日志
global mlog
mlog = myLog(module_name)
modelobj = scan(module_name)
dataDir = modelobj['dpath']
# 导入URL
luobj = modelobj['urlmodel']
urlmodel = luobj.urlput
for i in range(urlWorks):
product = Thread(target=urlmodel, args=(uin, dataDir, mlog))
product.setDaemon(True)
product.start()
threads.append(product)
# 判断逻辑
for i in range(dealWorks):
cusumer = mThread(uout, modelobj, mlog)
cusumer.setDaemon(True)
cusumer.start()
threads.append(cusumer)
return threads
def plot_scan_2d_surface(X, y, function, params):
listform = params.items()
train, test = scan(X, y, function, params)
surface(test)
return train, test