def test_equals(self):
data = 123
filter = Filter(equals=123)
self.assertTrue(filter(data))
data = '123'
self.assertTrue(filter(data))
filter = Filter(neq='abc')
self.assertTrue(filter(data))
def meow(days=None,
dates=None,
limit=None,
short=None,
fail=True,
exclude=None,
job_type=None,
down_path=config.DOWNLOAD_PATH,
periodic=False,
):
"""
This function actually runs the whole work,
you can import it anywhere and run with parameters:
:param days: how many days history to take, usually 7 (week) is enough
:param dates: specific dates in format ["%m-%d", ..]: ['04-15', '05-02']
:param limit: limit overall amount of jobs to analyze
:param short: analyze only this type of jobs,
accepts short name: "ha","upgrades","nonha"
:param fail: whether analyze and print only failed jobs (true by default)
:param exclude: exclude specific job type: "gate-tripleo-ci-f22-containers"
:param job_type: include only this job type (like short, but accepts
full name): "gate-tripleo-ci-f22-nonha"
:param down_path: path on local system to save all jobs files there
:param periodic: if take periodic (periodic=True) or patches (False)
:return: parsed jobs data, ready for printing to HTML or console
"""
if not periodic:
g = Gerrit(period=days)
#gerrit = g.get_project_patches(config.PROJECTS)
# Dump gerrit data for investigation
#with open("/tmp/gerrit", "w") as f:
# f.write(json.dumps(gerrit))
# If debug mode
with open("/tmp/gerrit", "r") as f:
gerrit = json.loads(f.read())
jobs = (job for patch in gerrit for job in Patch(patch).jobs)
else:
jobs = (job
for url in config.PERIODIC_URLS
for job in Periodic(
url, down_path=down_path, limit=limit).jobs)
f = Filter(
jobs,
days=days,
dates=dates,
limit=limit,
short=short,
fail=fail,
exclude=exclude,
job_type=job_type,
periodic=periodic
)
filtered = f.run()
ready = []
for job in filtered:
ready.append(analyze(job, down_path=down_path))
return ready
def split_lines(binary,
contour,
min_size,
thresh=0.5,
roi=False,
offset=(0, 0)):
if not roi:
roi_slice = utils.get_roi_slice(contour, offset)
binary = binary[roi_slice]
# cv2.imshow("bin", binary)
# ACHTUNG: may not work correct if `roi` is not correct contour's roi
binary = binary & np.array(utils.contour_mask(contour), dtype=bool)
means = binary.mean(axis=1)
# THRESH_INV
means[means < thresh] = -1
means[means >= thresh] = 0
means[means == -1] = 1
# Find where gaps starts and ends
means = np.pad(means, 1, 'constant')
means = means - np.roll(means, 1)
begins = (means == 1).nonzero()[0] - 1 # -1 caused by pad
ends = (means == -1).nonzero()[0] - 1
# Getting ars to cut
args = (begins + ends) / 2
args = args[args >= min_size]
args = args[args <= means.shape[0] - min_size]
if args.shape[0] == 0:
return [contour]
lines = []
prev = 0
args = np.append(args, means.shape[0] - 2)
for arg in args: # TODO: make this code reusable for fragmentation
line_roi = binary[prev:arg, :]
curr_off = (contour.rect.left, contour.rect.top + prev)
conts = [
Contour(c) for c in cv2.findContours(line_roi,
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE,
offset=curr_off)[0]
]
lines.extend(conts)
prev = arg
flt = Filter()
flt.add_filter("min_area", min_size * min_size)
lines = flt.filter(lines)
if len(lines) <= 1:
return [contour]
return lines
def initFilter():
global deffs, chain, fs
chain = FilterChain()
chain._filters.append(
Filter(FilterType.LShelving, LOW_EQ, 0, 1, enabled=True))
# chain._filters.append(Filter(FilterType.HShelving, deffs[4], 0, 1, enabled = True))
# chain._filters.append(Filter(FilterType.Peak, deffs[0], 0, 1, enabled = True))
chain._filters.append(Filter(FilterType.Peak, HIGH_EQ, 0, 1, enabled=True))
# chain._filters.append(Filter(FilterType.LPButter, deffs[3], 0, 1, enabled = True))
# chain._filters.append(Filter(FilterType.HPButter, deffs[3], 0, 1, enabled = True))
chain.reset()
def initFilter(self):
self.chain = FilterChain()
self.chain._filters.append(
Filter(FilterType.LShelving, self.LOW_EQ, 0, 1, enabled=True))
# self.chain._filters.append(Filter(FilterType.HShelving, deffs[4], 0, 1, enabled = True))
# self.chain._filters.append(Filter(FilterType.Peak, deffs[0], 0, 1, enabled = True))
self.chain._filters.append(
Filter(FilterType.Peak, self.HIGH_EQ, 0, 1, enabled=True))
# self.chain._filters.append(Filter(FilterType.LPButter, deffs[3], 0, 1, enabled = True))
# self.chain._filters.append(Filter(FilterType.HPButter, deffs[3], 0, 1, enabled = True))
self.chain.reset()
def test_melif(self):
data, target = self.basehock['X'], self.basehock['Y']
_filters = [
Filter('GiniIndex', cutting_rule=GLOB_CR["Best by value"](0.4)),
# Filter('FitCriterion', cutting_rule=GLOB_CR["Best by value"](0.0)),
Filter(GLOB_MEASURE["FRatio"](data.shape[1]),
cutting_rule=GLOB_CR["Best by value"](0.6)),
Filter('InformationGain',
cutting_rule=GLOB_CR["Best by value"](-0.4))
]
melif = Melif(_filters, f1_score)
melif.fit(data, target)
estimator = SVC()
melif.run(GLOB_CR['K best'](50), estimator)
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
clip = VideoFileClip("project_video_short3.mp4")
cnt = 0
stop_frame_num = 113
for img in clip.iter_frames():
cnt += 1
if (cnt == stop_frame_num):
if img.shape[2] == 4:
img = img[:, :, :3]
ret = filter.pipepine(img)
plt.figure(figsize=(16, 10))
plt.imshow(filter.diagScreen)
plt.subplots_adjust(left=0.03, bottom=0.03, right=1, top=1)
plt.show()
def updateFilter(self, i, param, val):
oldf = self.chain._filters[i]
type = oldf._type
fc = oldf._fc
g = oldf._g
Q = oldf._Q
if param == Params.TYPE:
type = val
Q = 1
elif param == Params.F:
fc = int(self.nodes[i].ctrls[2].text()) * 2 / fs
elif param == Params.G:
g = float(self.nodes[i].ctrls[3].text())
elif param == Params.Q:
if type == FilterType.LPButter or type == FilterType.HPButter:
Q = val
elif type == FilterType.Peak:
Q = val / 10
elif type == FilterType.LShelving or FilterType.HShelving:
Q = val / 100
self.chain.updateFilt(i, Filter(type, fc, g, Q))
if param == Params.TYPE:
self.updateControls(i, type)
self.adjustSliderRange(i, type)
self.updateSliderLabel(i)
def filterQuestions(results):
global bot
for x in results["items"]:
englishWords = 0;
question = Question(x)
quality = Filter(question)
quality.calc()
print quality.score
if quality.score < 40:
message = ""
for y in quality.text:
message += "Words: `%i`, Chars: `%i`, Bold: `%i`, Italic: `%i`, Case=> Upper: `%i`, Lower: `%i`" % (y.get("words"), y.get("chars"), y.get("bold"), y.get("italic"), y.get("case").get("upper"), y.get("case").get("lower"))
bot.send("[tag:low-quality] ["+question.title+"]("+question.link+") - "+message)
def build_soft_fusion_kernel(loops, loop_chain_index):
"""
Build AST and :class:`Kernel` for a sequence of loops suitable to soft fusion.
"""
kernels = [l.kernel for l in loops]
asts = [k._ast for k in kernels]
base_ast, fuse_asts = dcopy(asts[0]), asts[1:]
base_fundecl = FindInstances(ast.FunDecl).visit(base_ast)[ast.FunDecl][0]
base_fundecl.body[:] = [ast.Block(base_fundecl.body, open_scope=True)]
for unique_id, _fuse_ast in enumerate(fuse_asts, 1):
fuse_ast = dcopy(_fuse_ast)
fuse_fundecl = FindInstances(
ast.FunDecl).visit(fuse_ast)[ast.FunDecl][0]
# 1) Extend function name
base_fundecl.name = "%s_%s" % (base_fundecl.name, fuse_fundecl.name)
# 2) Concatenate the arguments in the signature
base_fundecl.args.extend(fuse_fundecl.args)
# 3) Uniquify symbols identifiers
fuse_symbols = SymbolReferences().visit(fuse_ast)
for decl in fuse_fundecl.args:
for symbol, _ in fuse_symbols[decl.sym.symbol]:
symbol.symbol = "%s_%d" % (symbol.symbol, unique_id)
# 4) Concatenate bodies
base_fundecl.body.extend(
[ast.FlatBlock("\n\n// Fused kernel: \n\n")] +
[ast.Block(fuse_fundecl.body, open_scope=True)])
# Eliminate redundancies in the /fused/ kernel signature
Filter().kernel_args(loops, base_fundecl)
return Kernel(kernels, base_ast, loop_chain_index)
def __compare_measure__(self, measure_name, data):
data, target = data['X'], data['Y']
start_time = time.time()
custom = lambda x, y: np.sum(x + y, axis=1)
f = Filter(custom, GLOB_CR["K best"](6))
res = f.run(
data, target
) # Filter(measure_name, GLOB_CR["K best"](6)).run(data, target)
print("ITMO_FS time --- %s seconds ---" % (time.time() - start_time))
start_time = time.time()
res = SelectKBest(GLOB_MEASURE[measure_name],
k=6).fit_transform(data, target)
print("SKLEARN time --- %s seconds ---" % (time.time() - start_time))
print(data.shape, '--->', res.shape)
def fetch_stocks(self, params):
''' if params==all fetch all stocks get_all_categories'''
filter = Filter()
parser = Parse()
stocklist = []
if params=='all':
cats = filter.get_all_categories()
for cat in cats:
params = [('sc', cat)]
try:
stocklist.extend(self.fetch_stocks(params))
except Exception, e:
print cat
print e
#print stocklist
print 'exited prematurely'
exit()
def run():
for aux in range(len(list_departaments)):
departament = DataSourceDepartament.read(
"https://matriculaweb.unb.br/graduacao/oferta_dep.aspx?cod=" +
str(list_departaments_numbers[aux]), list_departaments[aux])
DataSinkDepartament.write(departament, list_departaments[aux])
for new_aux in range(len(list_departaments_cods)):
subjects = DataSourceSubject.read(
"https://matriculaweb.unb.br/graduacao/oferta_dis.aspx?cod=" +
str(list_departaments_cods[new_aux]), new_aux)
DataSinkSubject.write(subjects)
subjects_darcy = DataSourceDepartament.read(
"https://matriculaweb.unb.br/graduacao/oferta_dep.aspx?cod=" +
str(list_departaments_numbers[3]), list_departaments[3])
subjects_darcy_cods = subjects_darcy.list_cods
subjects_darcy_initials = subjects_darcy.list_initials
size_subjects_darcy = 76
for aux in range(size_subjects_darcy):
subject = DataSourceSubjectsDarcy.read(
"https://matriculaweb.unb.br/graduacao/oferta_dis.aspx?cod=" +
str(subjects_darcy_cods[aux]), subjects_darcy_initials[aux],
subjects_darcy_cods[aux])
DataSinkSubject.write(subject)
for aux in range(len(list_campi_numbers)):
request = requests.get(
"https://matriculaweb.unb.br/graduacao/curso_rel.aspx?cod=" +
str(list_campi_numbers[aux]))
soup = BeautifulSoup(request.content, "html.parser")
courses = Courses()
list_content = []
for tr in soup.find_all('tr'):
for td in tr.find_all('td'):
list_content.append(td.text)
list_content = Filter.blank_space(list_content)
list_content = Filter.upper_words(list_content)
list_content = Filter.remove_accents(list_content)
list_cods = Filter.remove_vogals(list_content)
list_modalities = Filter.get_modalities(list_content)
list_shift = Filter.get_shift(list_content)
list_names = Filter.get_names(list_content)
courses.codes = list_cods
courses.modalities = list_modalities
courses.list_shift = list_shift
courses.names = list_names
DataSinkCourse.write(courses, list_departaments[aux])
def test_where(self):
l = ['ab', 'bc', 'ca', 'AB']
self.assertListEqual(SequenceChain(l).where().reveal(), l)
self.assertListEqual(
SequenceChain(l).where(Filter(eq='ab')).reveal(), ['ab'])
self.assertListEqual(
SequenceChain(l).where(Filter(contains='a')).reveal(),
['ab', 'ca'])
self.assertListEqual(
SequenceChain(l).where(Filter(icontains='a')).reveal(),
['ab', 'ca', 'AB'])
self.assertListEqual(
SequenceChain(l).where(Filter(istartswith='a')).reveal(),
['ab', 'AB'])
self.assertListEqual(
SequenceChain(l).where(Filter(contains='a',
contains_='b')).reveal(),
['ab', 'bc', 'ca'])
def mouseMoveEvent(self, e):
QFrame.mouseMoveEvent(self, e)
if self.dragged:
pos, i = e.pos(), self.focused
fc, g = fromPixelCords(self.width(), self.height(), pos, self.xaxis, self.raxis)
old = self.parent().chain._filters[i]
if old._type not in (FilterType.Peak, FilterType.LShelving, FilterType.HShelving):
g = 0
self.parent().chain.updateFilt(i,Filter(old._type, fc * 2 / fs, g, Q = old._Q))
self.updateHandles()
self.parent().updateChainTF()
self.update()
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
#filter.predict_batch(image_path=glob("./labeled_data_smallset/vehicles_smallset/**/*.*"))
#filter.predict_batch(image_path=filter.test_clf_image_paths)
frame = None
cnt = 0
for path in filter.test_video_images_path:
cnt += 1
if frame != None and cnt == frame:
image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
final_image = filter.pipepine(image)
plt.imshow(final_image)
plt.show()
break
elif frame == None:
image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
final_image = filter.pipepine(image)
plt.imshow(final_image)
plt.show()
# image_res, centroids_and_sizes = filter.sliding_box_multi_level(image, level=2)
"""
def updateFilter(self, i, fc, g, Q):
oldf = self.chain._filters[i]
type = oldf._type
# print oldf._type, oldf._fc, oldf._g, oldf._Q
# fc_val = fc * 2 / fs
# print fc_val, g, Q
f = Filter(type, fc, g, Q)
self.chain.updateFilt(i, f)
# chain.changeFilt(i, type, fc, g, Q)
self.chain.reset()
def updateFilter(i, fc, g, Q):
global chain
global fs
oldf = chain._filters[i]
type = oldf._type
# print oldf._type, oldf._fc, oldf._g, oldf._Q
# fc_val = fc * 2 / fs
# print fc_val, g, Q
f = Filter(type, fc, g, Q)
chain.updateFilt(i, f)
# chain.changeFilt(i, type, fc, g, Q)
chain.reset()
def parse(input):
"""
Trying to parse input. Depending on it
will return Filter or Event
:param input:
:return:
"""
input = str(input)
try:
message, category, to = re.findall('^(.*)\s#(\S*)\s@(\S*)$',
input)[0]
return Event(message, category, to, datetime.datetime.now())
except IndexError:
pass
try:
return Filter.create(input)
except ValueError as e:
raise ValueError("Input is not valid: \n{}".format(str(e)))
def scrape(data):
us_apps_filter = Filter(
field='app_store_url',
op=operator.contains,
value='/us/'
)
filter_chain = FilterChain()
filter_chain.add_filter(us_apps_filter)
us_apps = filter_chain.filter(data)
gathered_data = gather_data(us_apps)
spanish_and_tagalog_filter = ListContainedinListFilter(
field='languages',
op=operator.contains,
value=[u'Spanish', u'Tagalog']
)
filter_chain = FilterChain()
filter_chain.add_filter(spanish_and_tagalog_filter)
spanish_and_tagalog_data = filter_chain.filter(gathered_data)
insta_in_name_filter = CaseInsensitiveStringFilter(
field='name',
op=operator.contains,
value='insta'
)
filter_chain = FilterChain()
filter_chain.add_filter(insta_in_name_filter)
insta_in_name_data = filter_chain.filter(gathered_data)
filtered_data = {
'apps_in_spanish_and_tagalog': [_d.get('app_identifier') for _d in spanish_and_tagalog_data],
'apps_with_insta_in_name': [_d.get('app_identifier') for _d in insta_in_name_data]
}
write_json_to_file(filtered_data, 'filtered_apps.json')
write_json_to_file(gathered_data, 'apps.json')
def build_dfgp(self, filter_type):
"""Create DFGP based on internal settings. Creation ordering: build,
find loops and parallelisms, and the filter.
Args:
filter_type (str): Toggle how dfgp is filtered.
"""
# print settings
print("DFGP - settings > {}:{}:{}".format(
self.parallelisms_threshold, self.percentile_frequency_threshold,
filter_type))
# build
self.build()
loops = self.detect_loops_simple()
loops_extended = self.detect_loops_extended(loops)
self.detect_parallelisms(loops, loops_extended)
# filter
if filter_type == 'FWG':
Filter.with_guarantees(self, self.percentile_frequency_threshold)
elif filter_type == 'WTH':
Filter.with_threshold(self, self.percentile_frequency_threshold)
elif filter_type == 'STD':
Filter.standard(self)
def fetch_stocks(self, params):
filter = Filter()
parser = Parse()
url = filter.build_query_string(params)
results = parser.parse(url, [])
def test_compound_filter(varset_message, varset_callid_message, agentcalled_message):
message_filter = Filter(event="varset", variable="call_id")
assert not message_filter.match(varset_message)
assert message_filter.match(varset_callid_message)
assert not message_filter.match(agentcalled_message)
def test_basic_filter(varset_message, varset_callid_message, agentcalled_message):
message_filter = Filter(event="varset")
assert message_filter.match(varset_message)
assert message_filter.match(varset_callid_message)
assert not message_filter.match(agentcalled_message)
def test_filter(self):
data, target = load_iris(True)
res = Filter("SpearmanCorr",
GLOB_CR["Best by value"](0.9999)).run(data, target)
print("SpearmanCorr:", data.shape, '--->', res.shape)
def __test_mrmr(cls, data, target):
n = data.shape[1] / 2
res = Filter(GLOB_MEASURE["MrmrDiscrete"](n),
GLOB_CR["Best by value"](0.0)).run(data, target)
print("Mrmr:", data.shape, '--->', res.shape)
from astropy.table import Table
from spectrum import Spectrum
from filters import Filter
from diagram import ColourColourDiagram
import matplotlib.pyplot as plt
table = Table.read('sdss_data.fits') # magnitudes from SDSS so that we can classify our spectrum
stars = table[table['class'] == 'STAR']
qsos = table[table['class'] == 'QSO']
# read in the relevant filter curves
u_filter = Filter.from_file('sdss_filters/sdss-u.fits')
g_filter = Filter.from_file('sdss_filters/sdss-g.fits')
r_filter = Filter.from_file('sdss_filters/sdss-r.fits')
# Now define our colours using filters
# Python allows us to decide how objects use the `-` operator. Look at `Filter` to see how it works
ug_colour = u_filter - g_filter
gr_colour = g_filter - r_filter
star_galaxy_diagram = ColourColourDiagram(ug_colour, gr_colour) # make a colour-colour diagram using the two colours above
star_galaxy_diagram.plot_table(stars, c='b', alpha=0.2, s=1, label='stars') # plot the known stars using the `plot_table` method
star_galaxy_diagram.plot_table(qsos, c='r', alpha=0.2, s=1, label='QSOs') # plot the known qsos using the `plot_table` method
spectrum = Spectrum.from_file('spec-0266-51602-0003.fits') # read the spectrum into the `Spectrum` object
star_galaxy_diagram.plot_spectrum(spectrum, c='k', s=50) # plot the spectrum, where all the maths is behind the scenes
# So you can see that it is now really easy to plot any additional spectra we have without changing your code
plt.legend()
plt.show()
def render_POST(self, request):
"""Handle POST requests."""
global master, log, key, agents, filters, lock
log.info(request.path)
reqJSON = json.loads(request.content.getvalue())
if request.path == "/handshake": # New agent initializes secret key
# Retrieve public key
agent_pub = long(reqJSON['key'])
# Parse agent address
agent_addr = request.getClientIP() + ":" + str(reqJSON['port'])
# Find the agent
ret = ""
lock.acquire()
try:
for x in range(0, len(agents)):
if agents[x].getName() == reqJSON['name']: # If found
if not agents[x]: # If was dead
log.info("Recognized new agent: " +
agents[x].getName() + " is now alive.")
# Turn on agent and generate secret key
agents[x].setStatus(True)
agents[x].generateKey(agent_pub)
log.debug(agents[x].getKey())
ret = agents[x].getPublicKey()
finally:
lock.release()
# If agent was not recognized
if not ret:
temp = Agent(agent_addr, reqJSON['name'], True)
temp.generateKey(agent_pub)
ret = temp.getPublicKey()
log.debug(temp.getKey())
lock.acquire()
try:
agents.append(temp)
finally:
lock.release()
log.info("Recognized new agent: " + temp.getName() +
" is now alive.")
dumpContent()
# Return master's public key
return str(ret)
elif request.path == "/newagent" and request.getClientIP(
) == "127.0.0.1":
a = Agent(reqJSON['ip'] + ":" + reqJSON['port'], reqJSON['name'],
True)
shakeHands(a)
lock.acquire()
try:
agents.append(a)
finally:
lock.release()
if a:
init_one(a, log)
dumpContent()
log.info('Agent \'' + reqJSON['name'] +
'\' has been added from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/delagent" and request.getClientIP(
) == "127.0.0.1":
lock.acquire()
try:
for x in range(0, len(agents)):
if agents[x].getName() == reqJSON['name']:
del agents[x]
break
finally:
lock.release()
dumpContent()
log.info('Agent \'' + reqJSON['name'] +
'\' has been deleted from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/newfilter" and request.getClientIP(
) == "127.0.0.1":
lock.acquire()
try:
filters[reqJSON['name']] = Filter(reqJSON['filt'])
for agent in agents:
if agent and filters[reqJSON['name']].isUsed(
agent.getName()):
init_one(agent, log)
finally:
lock.release()
dumpContent()
log.info('Filter \'' + reqJSON['name'] +
'\' has been added from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
elif request.path == "/delfilter" and request.getClientIP(
) == "127.0.0.1":
reloads = []
for agent in agents:
lock.acquire()
try:
if agent and filters[reqJSON['name']].isUsed(
agent.getName()):
reloads.append(agent)
else:
lock.release()
finally:
lock.release()
lock.acquire()
try:
del filters[reqJSON['name']]
finally:
lock.release()
dumpContent()
for agent in reloads:
init_one(agent, log)
log.info('Filter \'' + reqJSON['name'] +
'\' has been deleted from web manager.')
request.setHeader("content-type", "application/json")
return json.JSONEncoder().encode({'success': True})
else:
request.setResponseCode(501)
return ""
def main():
filter = Filter(model_file="model.p", scaler_file="scaler.p")
image = cv2.imread(filter.ex.car_paths[0])
filter.extract_half_image_hog(image)
def _filter(self, loops):
return Filter().loop_args(loops).values()
def render_POST(self, request):
global master, key, filters
log.debug(request.path)
if request.path == "/handshake": # Generates new key. All other POST bodies are encoded with this secret key.
key.generateKey(long(request.content.getvalue()))
log.debug(key.getKey())
log.debug("Handshake complete. Secret key generated.")
return str(key.publicKey)
log.debug(util.decode(request.content.getvalue(), key.getKey()))
reqJSON = json.loads(util.decode(request.content.getvalue(), key.getKey())) # Decode POST body
if request.path[:5] == "/init":
if request.path == "/init":
log.info("Master is online. Initial retrieval of DNS settings begun.")
# Remove all conf files to create fresh ones
try:
os.remove(ns_file)
except:
log.warning("NS file (" + ns_file + ") has already been deleted.")
try:
os.remove(a_file)
except:
log.warning("A file (" + a_file + ") has already been deleted.")
try:
os.remove(ptr_file)
except:
log.warning("PTR file (" + ptr_file + ") has already been deleted.")
# Create new conf files
ns_f = open(ns_file, 'a')
a_f = open(a_file, 'a')
ptr_f = open(ptr_file, 'a')
# Write headers to each new conf file
ns_f.write("######### NS Records #########\n\n")
a_f.write("######### A Records #########\n\n")
ptr_f.write("######### PTR Records #########\n\n")
ns_f.close()
a_f.close()
ptr_f.close()
filters = {}
return json.dumps({'success': True})
# Endpoints to initialize each record type
if request.path[5:] == '/ns':
communicator.receiveFromFile(ns_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[5:] == '/a':
communicator.receiveFromFile(a_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[5:] == '/ptr':
communicator.receiveFromFile(ptr_file, reqJSON, filters)
return json.dumps({'success': True})
elif request.path[:4] == '/ns/':
if request.path[4:] == 'delete':
ns_records = communicator.fromJSON(reqJSON, log)['ns_records']
communicator.logSet(ns_records, log, "- ")
communicator.deleteFromFile(ns_records, ns_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[4:] == "add":
ns_records = communicator.fromJSON(reqJSON, log)['ns_records']
communicator.logSet(ns_records, log, "+ ")
communicator.addToFile(ns_records, ns_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path[:3] == '/a/':
if request.path[3:] == 'delete':
a_records = communicator.fromJSON(reqJSON, log)['a_records']
communicator.logSet(a_records, log, "- ")
communicator.deleteFromFile(a_records, a_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[3:] == "add":
a_records = communicator.fromJSON(reqJSON, log)['a_records']
communicator.logSet(a_records, log, "+ ")
communicator.addToFile(a_records, a_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path[:5] == '/ptr/':
if request.path[5:] == 'delete':
ptr_records = communicator.fromJSON(reqJSON, log)['ptr_records']
communicator.logSet(ptr_records, log, "- ")
communicator.deleteFromFile(ptr_records, ptr_file, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path[5:] == "add":
ptr_records = communicator.fromJSON(reqJSON, log)['ptr_records']
communicator.logSet(ptr_records, log, "+ ")
communicator.addToFile(ptr_records, ptr_file, filters, log)
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
elif request.path == '/filters':
log.debug("Filters:")
filters[reqJSON['code']] = Filter(reqJSON['filter'])
log.info("Added filter '" + reqJSON['code'] + "'")
log.debug(json.dumps(reqJSON['filter']))
request.responseHeaders.addRawHeader(b"content-type", b"application/json")
return json.dumps({'success': True})
elif request.path == '/reload':
log.info("DNS settings up to date. Reloading.")
util.execute("systemctl restart dnsmasq")
log.debug("DNSMasq reloaded.")
return json.dumps({'success': True})
else:
request.setResponseCode(501)
return ""
contents = os.listdir(directory)
files = []
for c in contents:
if os.path.isfile(os.path.join(directory, c)) and os.path.splitext(
c)[-1] in correct_extensions and os.path.splitext(
c)[0] not in banned:
files.append(os.path.join(directory, c))
elif os.path.isdir(os.path.join(directory,
c)) and c + "/" not in banned:
files += get_files_recursive(os.path.join(directory, c),
file_filter)
return files
file_filter = Filter()
"""
Enter directory for searching
"""
while True:
search_dir = input("Directory: ")
if os.path.exists(search_dir):
break
print("Directory doesn't exist!")
print("\n")
"""
Enter extensions to add
"""
while True:
ext = input("Enter extension (end): ")
if ext == "end":
from tensorflow.keras.models import model_from_json
import numpy as np
from datagenerator import FacialKeyPointsDataset
from filters import Filter
TYPE_OF_DATA_AND_MODEL = 'vector'
flags = {
"detect_faces": False,
"draw_keypts": False,
"filter": None,
"run": True
}
filters = [
Filter("images/filter1.png", 17, (25, 29), (2, 16), offset=(-10, 0)),
]
face_cascade = cv2.CascadeClassifier(
'detector_architectures/haarcascade_frontalface_default.xml')
datasetgen = FacialKeyPointsDataset(
csv_file='data/training_frames_keypoints.csv',
root_dir='data/training/',
normalization=TYPE_OF_DATA_AND_MODEL)
# load json and create model
json_file = open(
'models/model_{}_batchnorm_194.json'.format(TYPE_OF_DATA_AND_MODEL), 'r')
loaded_model_json = json_file.read()
json_file.close()
def find_text_lines(original,
thresh=0.02,
min_size=4,
max_size=12,
spacing=1.8,
min_length=1,
otsu=False,
x_only=False):
def _threshold_normal(_image, _thresh):
return np.uint8(
cv2.threshold(_image,
_thresh * utils.dtype_limits(_image)[1], 255,
cv2.THRESH_BINARY)[1])
def _threshold_otsu(_image, *args): # Careful with that axe, Eugene
return cv2.threshold(
np.uint8(_image / utils.dtype_limits(_image)[1] * 255.), 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]
if otsu:
get_thresh = _threshold_otsu
else:
get_thresh = _threshold_normal
def _find_text_preprocess(_image):
# _image = cv2.bilateralFilter(_image, -1, 64, 3)
# if gauss:
# _image = cv2.medianBlur(_image, 3)
# cv2.imshow('prep', _image)
if len(_image.shape) > 2:
_image = cv2.cvtColor(_image, cv2.COLOR_BGR2GRAY)
# image = cv2.equalizeHist(image)
return _image
def _get_diff(_image):
# _image = cv2.medianBlur(_image, 5)
# _image = cv2.GaussianBlur(_image, (3, 3), 5)
_image = utils.differentiate(_image, metric=utils.METRIC_MAX)
return utils.differentiate(_image, metric=utils.METRIC_MIN)
if spacing != 0:
ksize = int(min_size * spacing)
else:
ksize = 1
filters = Filter()
# filters.add_filter("outer")
filters.add_filter("min_area", min_size * min_size * min_length * 12)
# filters.add_filter("min_fill", 0.3)
filters.add_filter("min_aspect_ratio", 0.1)
# filters.add_filter("smoothness", 5)
# if diffs is None:
# diffs = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5, metric=utils.METRIC_SPLIT)
# if x_only:
# diff = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5, metric=utils.METRIC_SPLIT)[0]
# else:
# diff = utils.differentiate(np.float32(_find_text_preprocess(original)) / 255.,
# xkernel=5, ykernel=5)
# diff = utils.differentiate(np.float32(_find_text_preprocess(original) / 255.),
# metric=utils.METRIC_MIN)
diff = _get_diff(_find_text_preprocess(original))
ndiff = diff / diff.max()
binary = get_thresh(ndiff, thresh)
# cv2.imshow('diff', binary * 255)
# Connecting
connected = cv2.morphologyEx(binary,
cv2.MORPH_CLOSE,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(ksize, ksize)),
borderType=cv2.BORDER_CONSTANT,
borderValue=0)
# cv2.imshow('connected', connected)
# Filtering
binary = cv2.morphologyEx(
connected, cv2.MORPH_OPEN,
cv2.getStructuringElement(
cv2.MORPH_RECT,
utils.fix_kernel_size(int(min_size) - 1,
int(min_size) - 1)))
# cv2.imshow('bin', binary)
return filters.filter([
Contour(c, thresh) for c in cv2.findContours(
binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]
])
logToConsole = cfg.getboolean(section, 'logToConsole')
cfgLogLevel = cfg.get(section, 'cfgLogLevel')
log_max_size = cfg.getint(section, 'log_max_size')
log_max_backup = cfg.getint(section, 'log_max_backup')
section = 'Settings'
serverIP = cfg.get(section, 'serverIP')
port = cfg.getint(section, 'port')
shakeInterval = cfg.getint(section, 'handshake')
checkInterval = cfg.getint(section, 'statusCheck')
pollInterval = cfg.getint(section, 'poll')
filters = {}
json_fl = join(installPath, cfg.get(section, 'content'))
json_from_fl = json.load(open(json_fl))
for filt in json_from_fl['filters']:
filters[filt] = Filter(json_from_fl['filters'][filt])
agents = []
for agent in json_from_fl['agents']:
agents.append(Agent(agent['addr'], agent['name']))
section = 'Email Settings'
email_sender = cfg.get(section, 'from')
receivers = cfg.get(section, 'receivers').split(',')
section = 'Database'
host = cfg.get(section, 'host')
db_port = cfg.getint(section, 'port')
user = cfg.get(section, 'user')
password = cfg.get(section, 'password')
db = cfg.get(section, 'db')
tr_label_ = rg.transform(tr_label_.values, False)
#print rg.n_grid
print 'train...'
est = RandomForestClassifier(
n_jobs=-1,
n_estimators = 100,
max_features='sqrt',
bootstrap=True,
criterion='gini'
).fit(tr_feature_.values, tr_label_)
print 'predict...'
te_pred = est.predict_proba(te_feature_.values)
fl = Filter()
top_n = 1
te_pred_ = te_pred.copy()
pred_idx = np.argsort(-te_pred, axis=1)
for i, idx in enumerate(pred_idx[:, top_n:]):
te_pred_[i, idx] = 0.
z = np.sum(te_pred_, axis=1)
z.shape = len(z),1
te_pred_ = te_pred_ / z
te_pred_ = te_pred_.dot(rg.grid_center)
te_pred_ = fl.mean_filter(te_pred_, 11, 10)
error += [disney.distance(pt1, pt2) for pt1, pt2 in zip(te_pred_, te_label_.values)]
def filter_videos(self, filters):
self.filtered_playlists = Filter.filter_all(filters, self.playlists)
