def test_equals(self):
data = 123
filter = Filter(equals=123)
self.assertTrue(filter(data))
data = '123'
self.assertTrue(filter(data))
filter = Filter(neq='abc')
self.assertTrue(filter(data))
def initFilter(self):
self.chain = FilterChain()
self.chain._filters.append(
Filter(FilterType.LShelving, self.LOW_EQ, 0, 1, enabled=True))
# self.chain._filters.append(Filter(FilterType.HShelving, deffs[4], 0, 1, enabled = True))
# self.chain._filters.append(Filter(FilterType.Peak, deffs[0], 0, 1, enabled = True))
self.chain._filters.append(
Filter(FilterType.Peak, self.HIGH_EQ, 0, 1, enabled=True))
# self.chain._filters.append(Filter(FilterType.LPButter, deffs[3], 0, 1, enabled = True))
# self.chain._filters.append(Filter(FilterType.HPButter, deffs[3], 0, 1, enabled = True))
self.chain.reset()
def initFilter():
global deffs, chain, fs
chain = FilterChain()
chain._filters.append(
Filter(FilterType.LShelving, LOW_EQ, 0, 1, enabled=True))
# chain._filters.append(Filter(FilterType.HShelving, deffs[4], 0, 1, enabled = True))
# chain._filters.append(Filter(FilterType.Peak, deffs[0], 0, 1, enabled = True))
chain._filters.append(Filter(FilterType.Peak, HIGH_EQ, 0, 1, enabled=True))
# chain._filters.append(Filter(FilterType.LPButter, deffs[3], 0, 1, enabled = True))
# chain._filters.append(Filter(FilterType.HPButter, deffs[3], 0, 1, enabled = True))
chain.reset()
def test_melif(self):
data, target = self.basehock['X'], self.basehock['Y']
_filters = [
Filter('GiniIndex', cutting_rule=GLOB_CR["Best by value"](0.4)),
# Filter('FitCriterion', cutting_rule=GLOB_CR["Best by value"](0.0)),
Filter(GLOB_MEASURE["FRatio"](data.shape[1]),
cutting_rule=GLOB_CR["Best by value"](0.6)),
Filter('InformationGain',
cutting_rule=GLOB_CR["Best by value"](-0.4))
]
melif = Melif(_filters, f1_score)
melif.fit(data, target)
estimator = SVC()
melif.run(GLOB_CR['K best'](50), estimator)
def build_soft_fusion_kernel(loops, loop_chain_index):
"""
Build AST and :class:`Kernel` for a sequence of loops suitable to soft fusion.
"""
kernels = [l.kernel for l in loops]
asts = [k._ast for k in kernels]
base_ast, fuse_asts = dcopy(asts[0]), asts[1:]
base_fundecl = FindInstances(ast.FunDecl).visit(base_ast)[ast.FunDecl][0]
base_fundecl.body[:] = [ast.Block(base_fundecl.body, open_scope=True)]
for unique_id, _fuse_ast in enumerate(fuse_asts, 1):
fuse_ast = dcopy(_fuse_ast)
fuse_fundecl = FindInstances(
ast.FunDecl).visit(fuse_ast)[ast.FunDecl][0]
# 1) Extend function name
base_fundecl.name = "%s_%s" % (base_fundecl.name, fuse_fundecl.name)
# 2) Concatenate the arguments in the signature
base_fundecl.args.extend(fuse_fundecl.args)
# 3) Uniquify symbols identifiers
fuse_symbols = SymbolReferences().visit(fuse_ast)
for decl in fuse_fundecl.args:
for symbol, _ in fuse_symbols[decl.sym.symbol]:
symbol.symbol = "%s_%d" % (symbol.symbol, unique_id)
# 4) Concatenate bodies
base_fundecl.body.extend(
[ast.FlatBlock("\n\n// Fused kernel: \n\n")] +
[ast.Block(fuse_fundecl.body, open_scope=True)])
# Eliminate redundancies in the /fused/ kernel signature
Filter().kernel_args(loops, base_fundecl)
return Kernel(kernels, base_ast, loop_chain_index)
def updateFilter(self, i, param, val):
oldf = self.chain._filters[i]
type = oldf._type
fc = oldf._fc
g = oldf._g
Q = oldf._Q
if param == Params.TYPE:
type = val
Q = 1
elif param == Params.F:
fc = int(self.nodes[i].ctrls[2].text()) * 2 / fs
elif param == Params.G:
g = float(self.nodes[i].ctrls[3].text())
elif param == Params.Q:
if type == FilterType.LPButter or type == FilterType.HPButter:
Q = val
elif type == FilterType.Peak:
Q = val / 10
elif type == FilterType.LShelving or FilterType.HShelving:
Q = val / 100
self.chain.updateFilt(i, Filter(type, fc, g, Q))
if param == Params.TYPE:
self.updateControls(i, type)
self.adjustSliderRange(i, type)
self.updateSliderLabel(i)
def test_where(self):
l = ['ab', 'bc', 'ca', 'AB']
self.assertListEqual(SequenceChain(l).where().reveal(), l)
self.assertListEqual(
SequenceChain(l).where(Filter(eq='ab')).reveal(), ['ab'])
self.assertListEqual(
SequenceChain(l).where(Filter(contains='a')).reveal(),
['ab', 'ca'])
self.assertListEqual(
SequenceChain(l).where(Filter(icontains='a')).reveal(),
['ab', 'ca', 'AB'])
self.assertListEqual(
SequenceChain(l).where(Filter(istartswith='a')).reveal(),
['ab', 'AB'])
self.assertListEqual(
SequenceChain(l).where(Filter(contains='a',
contains_='b')).reveal(),
['ab', 'bc', 'ca'])
def split_lines(binary,
contour,
min_size,
thresh=0.5,
roi=False,
offset=(0, 0)):
if not roi:
roi_slice = utils.get_roi_slice(contour, offset)
binary = binary[roi_slice]
# cv2.imshow("bin", binary)
# ACHTUNG: may not work correct if `roi` is not correct contour's roi
binary = binary & np.array(utils.contour_mask(contour), dtype=bool)
means = binary.mean(axis=1)
# THRESH_INV
means[means < thresh] = -1
means[means >= thresh] = 0
means[means == -1] = 1
# Find where gaps starts and ends
means = np.pad(means, 1, 'constant')
means = means - np.roll(means, 1)
begins = (means == 1).nonzero()[0] - 1 # -1 caused by pad
ends = (means == -1).nonzero()[0] - 1
# Getting ars to cut
args = (begins + ends) / 2
args = args[args >= min_size]
args = args[args <= means.shape[0] - min_size]
if args.shape[0] == 0:
return [contour]
lines = []
prev = 0
args = np.append(args, means.shape[0] - 2)
for arg in args: # TODO: make this code reusable for fragmentation
line_roi = binary[prev:arg, :]
curr_off = (contour.rect.left, contour.rect.top + prev)
conts = [
Contour(c) for c in cv2.findContours(line_roi,
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE,
offset=curr_off)[0]
]
lines.extend(conts)
prev = arg
flt = Filter()
flt.add_filter("min_area", min_size * min_size)
lines = flt.filter(lines)
if len(lines) <= 1:
return [contour]
return lines
def updateFilter(self, i, fc, g, Q):
oldf = self.chain._filters[i]
type = oldf._type
# print oldf._type, oldf._fc, oldf._g, oldf._Q
# fc_val = fc * 2 / fs
# print fc_val, g, Q
f = Filter(type, fc, g, Q)
self.chain.updateFilt(i, f)
# chain.changeFilt(i, type, fc, g, Q)
self.chain.reset()
def mouseMoveEvent(self, e):
QFrame.mouseMoveEvent(self, e)
if self.dragged:
pos, i = e.pos(), self.focused
fc, g = fromPixelCords(self.width(), self.height(), pos, self.xaxis, self.raxis)
old = self.parent().chain._filters[i]
if old._type not in (FilterType.Peak, FilterType.LShelving, FilterType.HShelving):
g = 0
self.parent().chain.updateFilt(i,Filter(old._type, fc * 2 / fs, g, Q = old._Q))
self.updateHandles()
self.parent().updateChainTF()
self.update()
def __compare_measure__(self, measure_name, data):
data, target = data['X'], data['Y']
start_time = time.time()
res = Filter(measure_name, GLOB_CR["K best"](6)).run(data, target)
print("ITMO_FS time --- %s seconds ---" % (time.time() - start_time))
start_time = time.time()
res = SelectKBest(GLOB_MEASURE[measure_name],
k=6).fit_transform(data, target)
print("SKLEARN time --- %s seconds ---" % (time.time() - start_time))
print(data.shape, '--->', res.shape)
def updateFilter(i, fc, g, Q):
global chain
global fs
oldf = chain._filters[i]
type = oldf._type
# print oldf._type, oldf._fc, oldf._g, oldf._Q
# fc_val = fc * 2 / fs
# print fc_val, g, Q
f = Filter(type, fc, g, Q)
chain.updateFilt(i, f)
# chain.changeFilt(i, type, fc, g, Q)
chain.reset()
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
clip = VideoFileClip("project_video_short3.mp4")
cnt = 0
stop_frame_num = 113
for img in clip.iter_frames():
cnt += 1
if (cnt == stop_frame_num):
if img.shape[2] == 4:
img = img[:, :, :3]
ret = filter.pipepine(img)
plt.figure(figsize=(16, 10))
plt.imshow(filter.diagScreen)
plt.subplots_adjust(left=0.03, bottom=0.03, right=1, top=1)
plt.show()
def fetch_stocks(self, params):
''' if params==all fetch all stocks get_all_categories'''
filter = Filter()
parser = Parse()
stocklist = []
if params=='all':
cats = filter.get_all_categories()
for cat in cats:
params = [('sc', cat)]
try:
stocklist.extend(self.fetch_stocks(params))
except Exception, e:
print cat
print e
#print stocklist
print 'exited prematurely'
exit()
def scrape(data):
us_apps_filter = Filter(
field='app_store_url',
op=operator.contains,
value='/us/'
)
filter_chain = FilterChain()
filter_chain.add_filter(us_apps_filter)
us_apps = filter_chain.filter(data)
gathered_data = gather_data(us_apps)
spanish_and_tagalog_filter = ListContainedinListFilter(
field='languages',
op=operator.contains,
value=[u'Spanish', u'Tagalog']
)
filter_chain = FilterChain()
filter_chain.add_filter(spanish_and_tagalog_filter)
spanish_and_tagalog_data = filter_chain.filter(gathered_data)
insta_in_name_filter = CaseInsensitiveStringFilter(
field='name',
op=operator.contains,
value='insta'
)
filter_chain = FilterChain()
filter_chain.add_filter(insta_in_name_filter)
insta_in_name_data = filter_chain.filter(gathered_data)
filtered_data = {
'apps_in_spanish_and_tagalog': [_d.get('app_identifier') for _d in spanish_and_tagalog_data],
'apps_with_insta_in_name': [_d.get('app_identifier') for _d in insta_in_name_data]
}
write_json_to_file(filtered_data, 'filtered_apps.json')
write_json_to_file(gathered_data, 'apps.json')
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
#filter.predict_batch(image_path=glob("./labeled_data_smallset/vehicles_smallset/**/*.*"))
#filter.predict_batch(image_path=filter.test_clf_image_paths)
frame = None
cnt = 0
for path in filter.test_video_images_path:
cnt += 1
if frame != None and cnt == frame:
image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
final_image = filter.pipepine(image)
plt.imshow(final_image)
plt.show()
break
elif frame == None:
image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
final_image = filter.pipepine(image)
plt.imshow(final_image)
plt.show()
# image_res, centroids_and_sizes = filter.sliding_box_multi_level(image, level=2)
"""
contents = os.listdir(directory)
files = []
for c in contents:
if os.path.isfile(os.path.join(directory, c)) and os.path.splitext(
c)[-1] in correct_extensions and os.path.splitext(
c)[0] not in banned:
files.append(os.path.join(directory, c))
elif os.path.isdir(os.path.join(directory,
c)) and c + "/" not in banned:
files += get_files_recursive(os.path.join(directory, c),
file_filter)
return files
file_filter = Filter()
"""
Enter directory for searching
"""
while True:
search_dir = input("Directory: ")
if os.path.exists(search_dir):
break
print("Directory doesn't exist!")
print("\n")
"""
Enter extensions to add
"""
while True:
ext = input("Enter extension (end): ")
if ext == "end":
def render_POST(self, request):
global master, key, filters
log.debug(request.path)
if request.path == "/handshake": # Generates new key. All other POST bodies are encoded with this secret key.
key.generateKey(long(request.content.getvalue()))
log.debug(key.getKey())
log.debug("Handshake complete. Secret key generated.")
return str(key.publicKey)
log.debug(util.decode(request.content.getvalue(), key.getKey()))
reqJSON = json.loads(util.decode(request.content.getvalue(), key.getKey())) # Decode POST body
if request.path[:5] == "/init":
if request.path == "/init":
log.info("Master is online. Initial retrieval of DNS settings begun.")
# Remove all conf files to create fresh ones
try:
os.remove(ns_file)
except:
log.warning("NS file (" + ns_file + ") has already been deleted.")
try:
os.remove(a_file)
except:
log.warning("A file (" + a_file + ") has already been deleted.")
try:
os.remove(ptr_file)
except:
log.warning("PTR file (" + ptr_file + ") has already been deleted.")
# Create new conf files
ns_f = open(ns_file, 'a')
a_f = open(a_file, 'a')
ptr_f = open(ptr_file, 'a')
# Write headers to each new conf file
ns_f.write("######### NS Records #########\n\n")
a_f.write("######### A Records #########\n\n")
ptr_f.write("######### PTR Records #########\n\n")
ns_f.close()
a_f.close()
ptr_f.close()
filters = {}
return json.dumps({'success': True})
# Endpoints to initialize each record type
if request.path[5:] == '/ns':
communicator.receiveFromFile(ns_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[5:] == '/a':
communicator.receiveFromFile(a_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[5:] == '/ptr':
communicator.receiveFromFile(ptr_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[:4] == '/ns/':
if request.path[4:] == 'delete':
ns_records = communicator.fromJSON(reqJSON, log)['ns_records']
communicator.logSet(ns_records, log, "- ")
communicator.deleteFromFile(ns_records, ns_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[4:] == "add":
ns_records = communicator.fromJSON(reqJSON, log)['ns_records']
communicator.logSet(ns_records, log, "+ ")
communicator.addToFile(ns_records, ns_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path[:3] == '/a/':
if request.path[3:] == 'delete':
a_records = communicator.fromJSON(reqJSON, log)['a_records']
communicator.logSet(a_records, log, "- ")
communicator.deleteFromFile(a_records, a_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[3:] == "add":
a_records = communicator.fromJSON(reqJSON, log)['a_records']
communicator.logSet(a_records, log, "+ ")
communicator.addToFile(a_records, a_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path[:5] == '/ptr/':
if request.path[5:] == 'delete':
ptr_records = communicator.fromJSON(reqJSON, log)['ptr_records']
communicator.logSet(ptr_records, log, "- ")
communicator.deleteFromFile(ptr_records, ptr_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[5:] == "add":
ptr_records = communicator.fromJSON(reqJSON, log)['ptr_records']
communicator.logSet(ptr_records, log, "+ ")
communicator.addToFile(ptr_records, ptr_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path == '/filters':
log.debug("Filters:")
filters[reqJSON['code']] = Filter(reqJSON['filter'])
log.info("Added filter '" + reqJSON['code'] + "'")
log.debug(json.dumps(reqJSON['filter']))
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path == '/reload':
log.info("DNS settings up to date. Reloading.")
util.execute("systemctl restart dnsmasq")
log.debug("DNSMasq reloaded.")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
def test_iequals(self):
data = 'ABCDE'
filter = Filter(ieq='abcde')
self.assertTrue(filter(data))
def test_filter(self):
data, target = load_iris(True)
res = Filter("SpearmanCorr",
GLOB_CR["Best by value"](0.9999)).run(data, target)
print("SpearmanCorr:", data.shape, '--->', res.shape)
def __test_mrmr(cls, data, target):
n = data.shape[1] / 2
res = Filter(GLOB_MEASURE["MrmrDiscrete"](n),
GLOB_CR["Best by value"](0.0)).run(data, target)
print("Mrmr:", data.shape, '--->', res.shape)
def render_POST(self, request):
"""Handle POST requests."""
global master, log, key, agents, filters, lock
log.info(request.path)
reqJSON = json.loads(request.content.getvalue())
if request.path == "/handshake": # New agent initializes secret key
# Retrieve public key
agent_pub = long(reqJSON['key'])
# Parse agent address
agent_addr = request.getClientIP() + ":" + str(reqJSON['port'])
# Find the agent
ret = ""
lock.acquire()
try:
for x in range(0, len(agents)):
if agents[x].getName() == reqJSON['name']: # If found
if not agents[x]: # If was dead
log.info("Recognized new agent: " +
agents[x].getName() + " is now alive.")
# Turn on agent and generate secret key
agents[x].setStatus(True)
agents[x].generateKey(agent_pub)
log.debug(agents[x].getKey())
ret = agents[x].getPublicKey()
finally:
lock.release()
# If agent was not recognized
if not ret:
temp = Agent(agent_addr, reqJSON['name'], True)
temp.generateKey(agent_pub)
ret = temp.getPublicKey()
log.debug(temp.getKey())
lock.acquire()
try:
agents.append(temp)
finally:
lock.release()
log.info("Recognized new agent: " + temp.getName() +
" is now alive.")
dumpContent()
# Return master's public key
return str(ret)
elif request.path == "/newagent" and request.getClientIP(
) == "127.0.0.1":
a = Agent(reqJSON['ip'] + ":" + reqJSON['port'], reqJSON['name'],
True)
shakeHands(a)
lock.acquire()
try:
agents.append(a)
finally:
lock.release()
if a:
init_one(a, log)
dumpContent()
log.info('Agent \'' + reqJSON['name'] +
'\' has been added from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/delagent" and request.getClientIP(
) == "127.0.0.1":
lock.acquire()
try:
for x in range(0, len(agents)):
if agents[x].getName() == reqJSON['name']:
del agents[x]
break
finally:
lock.release()
dumpContent()
log.info('Agent \'' + reqJSON['name'] +
'\' has been deleted from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/newfilter" and request.getClientIP(
) == "127.0.0.1":
lock.acquire()
try:
filters[reqJSON['name']] = Filter(reqJSON['filt'])
for agent in agents:
if agent and filters[reqJSON['name']].isUsed(
agent.getName()):
init_one(agent, log)
finally:
lock.release()
dumpContent()
log.info('Filter \'' + reqJSON['name'] +
'\' has been added from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/delfilter" and request.getClientIP(
) == "127.0.0.1":
reloads = []
for agent in agents:
lock.acquire()
try:
if agent and filters[reqJSON['name']].isUsed(
agent.getName()):
reloads.append(agent)
else:
lock.release()
finally:
lock.release()
lock.acquire()
try:
del filters[reqJSON['name']]
finally:
lock.release()
dumpContent()
for agent in reloads:
init_one(agent, log)
log.info('Filter \'' + reqJSON['name'] +
'\' has been deleted from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
else:
request.setResponseCode(501)
return ""
logToConsole = cfg.getboolean(section, 'logToConsole')
cfgLogLevel = cfg.get(section, 'cfgLogLevel')
log_max_size = cfg.getint(section, 'log_max_size')
log_max_backup = cfg.getint(section, 'log_max_backup')
section = 'Settings'
serverIP = cfg.get(section, 'serverIP')
port = cfg.getint(section, 'port')
shakeInterval = cfg.getint(section, 'handshake')
checkInterval = cfg.getint(section, 'statusCheck')
pollInterval = cfg.getint(section, 'poll')
filters = {}
json_fl = join(installPath, cfg.get(section, 'content'))
json_from_fl = json.load(open(json_fl))
for filt in json_from_fl['filters']:
filters[filt] = Filter(json_from_fl['filters'][filt])
agents = []
for agent in json_from_fl['agents']:
agents.append(Agent(agent['addr'], agent['name']))
section = 'Email Settings'
email_sender = cfg.get(section, 'from')
receivers = cfg.get(section, 'receivers').split(',')
section = 'Database'
host = cfg.get(section, 'host')
db_port = cfg.getint(section, 'port')
user = cfg.get(section, 'user')
password = cfg.get(section, 'password')
db = cfg.get(section, 'db')
from tensorflow.keras.models import model_from_json
import numpy as np
from datagenerator import FacialKeyPointsDataset
from filters import Filter
TYPE_OF_DATA_AND_MODEL = 'vector'
flags = {
"detect_faces": False,
"draw_keypts": False,
"filter": None,
"run": True
}
filters = [
Filter("images/filter1.png", 17, (25, 29), (2, 16), offset=(-10, 0)),
]
face_cascade = cv2.CascadeClassifier(
'detector_architectures/haarcascade_frontalface_default.xml')
datasetgen = FacialKeyPointsDataset(
csv_file='data/training_frames_keypoints.csv',
root_dir='data/training/',
normalization=TYPE_OF_DATA_AND_MODEL)
# load json and create model
json_file = open(
'models/model_{}_batchnorm_194.json'.format(TYPE_OF_DATA_AND_MODEL), 'r')
loaded_model_json = json_file.read()
json_file.close()
def test_regexp(self):
data = 'xx10xx'
filter = Filter(search='\d')
self.assertTrue(filter(data))
filter = Filter(match='\d')
self.assertFalse(filter(data))
def fetch_stocks(self, params):
filter = Filter()
parser = Parse()
url = filter.build_query_string(params)
results = parser.parse(url, [])
def find_text_lines(original,
thresh=0.02,
min_size=4,
max_size=12,
spacing=1.8,
min_length=1,
otsu=False,
x_only=False):
def _threshold_normal(_image, _thresh):
return np.uint8(
cv2.threshold(_image,
_thresh * utils.dtype_limits(_image)[1], 255,
cv2.THRESH_BINARY)[1])
def _threshold_otsu(_image, *args): # Careful with that axe, Eugene
return cv2.threshold(
np.uint8(_image / utils.dtype_limits(_image)[1] * 255.), 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]
if otsu:
get_thresh = _threshold_otsu
else:
get_thresh = _threshold_normal
def _find_text_preprocess(_image):
# _image = cv2.bilateralFilter(_image, -1, 64, 3)
# if gauss:
# _image = cv2.medianBlur(_image, 3)
# cv2.imshow('prep', _image)
if len(_image.shape) > 2:
_image = cv2.cvtColor(_image, cv2.COLOR_BGR2GRAY)
# image = cv2.equalizeHist(image)
return _image
def _get_diff(_image):
# _image = cv2.medianBlur(_image, 5)
# _image = cv2.GaussianBlur(_image, (3, 3), 5)
_image = utils.differentiate(_image, metric=utils.METRIC_MAX)
return utils.differentiate(_image, metric=utils.METRIC_MIN)
if spacing != 0:
ksize = int(min_size * spacing)
else:
ksize = 1
filters = Filter()
# filters.add_filter("outer")
filters.add_filter("min_area", min_size * min_size * min_length * 12)
# filters.add_filter("min_fill", 0.3)
filters.add_filter("min_aspect_ratio", 0.1)
# filters.add_filter("smoothness", 5)
# if diffs is None:
# diffs = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5, metric=utils.METRIC_SPLIT)
# if x_only:
# diff = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5, metric=utils.METRIC_SPLIT)[0]
# else:
# diff = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5)
# diff = utils.differentiate(np.float32(_find_text_preprocess(original) / 255.),
# metric=utils.METRIC_MIN)
diff = _get_diff(_find_text_preprocess(original))
ndiff = diff / diff.max()
binary = get_thresh(ndiff, thresh)
# cv2.imshow('diff', binary * 255)
# Connecting
connected = cv2.morphologyEx(binary,
cv2.MORPH_CLOSE,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(ksize, ksize)),
borderType=cv2.BORDER_CONSTANT,
borderValue=0)
# cv2.imshow('connected', connected)
# Filtering
binary = cv2.morphologyEx(
connected, cv2.MORPH_OPEN,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(int(min_size) - 1,
int(min_size) - 1)))
# cv2.imshow('bin', binary)
return filters.filter([
Contour(c, thresh) for c in cv2.findContours(
binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]
])
def _filter(self, loops):
return Filter().loop_args(loops).values()
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
image = cv2.imread(filter.ex.car_paths[0])
filter.extract_half_image_hog(image)
def find_text(original=None,
diffs=None,
first_thresh=0.02,
second_thresh=0.04,
min_size=4,
max_size=12,
hspacing=1.8,
vspacing=0,
min_length=1,
offset=(0, 0),
otsu=False,
split=True,
max_box_rad=(32, 32),
default_box_rad=(2, 2),
ret=TEXT_CONTOURS_BOXES):
"""
Detects text areas on image regardless to its contents
:param hspacing: Maximal horizontal spacing for symbols connection (relative to min_size)
:param min_length: Minimal text length (relative to min_size)
"""
_debug = False
def _threshold_normal(_image, _thresh):
return np.uint8(
cv2.threshold(_image,
_thresh * utils.dtype_limits(_image)[1], 255,
cv2.THRESH_BINARY)[1])
def _threshold_otsu(_image, *args): # Careful with that axe, Eugene
return cv2.threshold(
np.uint8(_image / utils.dtype_limits(_image)[1] * 255.), 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]
if otsu:
get_thresh = _threshold_otsu
else:
get_thresh = _threshold_normal
def _find_text_preprocess(_image):
# image = cv2.bilateralFilter(image, -1, 64, 3)
_image = cv2.cvtColor(_image, cv2.COLOR_BGR2GRAY)
# image = cv2.equalizeHist(image)
return _image
assert (original is not None or diffs is not None)
if hspacing != 0:
ksize_h = int(min_size * hspacing)
else:
ksize_h = 1
if vspacing != 0:
ksize_v = int(min_size * vspacing)
else:
ksize_v = 1
filters = Filter()
filters.add_filter("outer")
filters.add_filter("min_area", min_size * min_size * min_length)
# filters.add_filter("min_fill", 0.5)
# filters.add_filter("min_aspect_ratio", min_length)
# if diffs is None:
# diffs = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5, metric=utils.METRIC_SPLIT)
if diffs is None:
diffs = utils.differentiate(
np.float32(_find_text_preprocess(original)) / 255.,
xkernel=5,
ykernel=5)
diffs = (diffs, diffs)
ndiffs = (diffs[0] / diffs[0].max(), diffs[1] / diffs[1].max())
if _debug:
cv2.imshow("raw bin", ndiffs[0] / ndiffs[0].max())
binary = get_thresh(ndiffs[0], first_thresh)
if _debug:
cv2.imshow("bin", binary)
# Remove too long vertical lines
sub = cv2.morphologyEx(
binary, cv2.MORPH_OPEN,
cv2.getStructuringElement(cv2.MORPH_RECT,
utils.fix_kernel_size(1, max_size)))
sub = cv2.morphologyEx(sub, cv2.MORPH_CLOSE,
cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
if _debug:
cv2.imshow("sub", sub)
binary = cv2.bitwise_xor(binary, sub)
# Connecting
connected = cv2.morphologyEx(binary,
cv2.MORPH_CLOSE,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(ksize_h, ksize_v)),
borderType=cv2.BORDER_CONSTANT,
borderValue=0)
if _debug:
cv2.imshow("connected", connected)
# Filtering
binary = cv2.morphologyEx(
connected, cv2.MORPH_OPEN,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(min_length * int(min_size) - 1,
int(min_size) - 1)))
if _debug:
cv2.imshow("connected and filtered", binary)
# 1st detect_angle, not accurate, with max_thresh, not using offset yet
conts = [
Contour(c, first_thresh) for c in cv2.findContours(
binary, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)[0]
]
filtered = filters.filter(conts)
# Preparing data for floodFill
if ret == TEXT_BOXES or ret == TEXT_CONTOURS_BOXES:
second_bin = (get_thresh(ndiffs[0], second_thresh),
get_thresh(ndiffs[1], second_thresh))
else:
second_bin = (get_thresh(ndiffs[0], second_thresh), None)
bin_for_ff = cv2.bitwise_or(second_bin[0] * 1., (connected / 255) * 2)
mask = cv2.bitwise_not(cv2.bitwise_or(binary, connected))
mask = np.pad(mask, 1, 'constant', constant_values=255) # add 1px. border
for c in filtered:
cv2.floodFill(bin_for_ff,
mask, (c.raw[0][0][0], c.raw[0][0][1]),
255,
flags=4)
bin_for_ff = np.uint8(
cv2.threshold(bin_for_ff, 1, 255, cv2.THRESH_BINARY)[1])
# bin_for_ff = cv2.morphologyEx(bin_for_ff, cv2.MORPH_OPEN,
# cv2.getStructuringElement(cv2.MORPH_RECT,
# utils.fix_kernel_size(min_length * int(min_size) - 1,
# int(min_size) - 1)))
if split:
bin_for_split = bin_for_ff.copy()
if _debug:
cv2.imshow("bin_for_ff", bin_for_ff)
# (offset[0] - 1) is a workaround for some kind of bug
# 2nd detect_angle, more accurate, with min_thresh and offset
conts = [
Contour(c, first_thresh)
for c in cv2.findContours(bin_for_ff,
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE,
offset=(offset[0] - 1, offset[1]))[0]
]
filtered = filters.filter(conts)
if not split:
splitted = filtered
else:
splitted = []
for c in filtered:
if c.rect.height >= 2 * min_size:
splitted.extend(
split_lines(bin_for_split,
c,
min_size,
offset=(-offset[0], -offset[1])))
else:
splitted.append(c)
if ret == TEXT_CONTOURS:
return splitted
else:
boxes = [
find_text_box(contour=c,
bdiff=second_bin,
max_rad=max_box_rad,
default_rad=default_box_rad,
offset=offset) for c in splitted
]
if ret == TEXT_BOXES:
return boxes
elif ret == TEXT_CONTOURS_BOXES:
return zip(splitted, boxes)
