def test_filter_simple(self):
res = [
'',
'',
'',
'cow',
'siamese',
'wonderland',
'foo',
'toothpaste',
'',
'',
'',
'milky',
'flight-manual',
'toothpick',
'',
None,
'',
'',
]
generated = []
filter = Filter(options)
for word in words:
generated.append(filter.filter(word))
self.assertEqual(generated, res)
class SearchHandler:
def __init__(self):
self.autoComplete = AutoComplete()
self.resultFilter = Filter()
self.googleCustom = GoogleCustomSearch()
self.googleScholar = GoogleScholar()
# returns a list of dictionaries for results
def search(self,
query: str,
from_sites: tuple = None,
ignore_sites: tuple = None) -> list:
args = dict()
results = list()
if from_sites:
args['from_sites'] = from_sites
if ignore_sites:
args['ignore_sites'] = ignore_sites
results.append(self.googleCustom.search(query))
scholar_results = self.googleScholar.search(query)
scholar_results.append('TRUSTED')
results.append(scholar_results)
filtered = self.resultFilter.filter(results, args)
return filtered
def test_filter_finish_same_as_stop(self):
words = [
'cow',
'foo',
'fish',
'flamingo',
'trampoline',
'apollo',
'cow',
'pancakes',
]
options = {'start': 'cow', 'stop': 'Flamingo', 'finish': 'flamingo'}
res = [
'',
'foo',
'fish',
None,
'',
'',
'',
'pancakes',
]
w = Filter(options)
filtered = []
for word in words:
filtered.append(w.filter(word))
self.assertEqual(res, filtered)
def _generateArtifactList(options):
# load configuration
logging.info("Loading configuration...")
config = Configuration()
config.load(options)
# build list
logging.info("Building artifact list...")
listBuilder = ArtifactListBuilder(config)
artifactList = listBuilder.buildList()
logging.debug("Generated list contents:")
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
logging.debug(" %s:%s", gat, version)
#filter list
logging.info("Filtering artifact list...")
listFilter = Filter(config)
artifactList = listFilter.filter(artifactList)
logging.debug("Filtered list contents:")
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
logging.debug(" %s:%s", gat, version)
logging.info("Artifact list generation done")
return artifactList
def do_stuff(train, messages):
# Open files for reading and writing
train_file = open(train, "rb")
messages_file = open(messages, "rb")
predictions_file = open("predictions.txt", "w")
# Create new filter and train it using the train-file
f = Filter()
f.train(train_file)
#filter the messages in messages_file, write results to predictions_file
f.filter(messages_file, predictions_file)
# Close all the files
train_file.close()
messages_file.close()
predictions_file.close()
def test_filter():
mappings = [{'source': 'title'}, {'source': 'description'}, {'source': 'adfasdf'}, {'no source': 'adfasdf'}]
filter = Filter(mappings)
result = {}
result['items'] = [{
'author': 'Fred',
'guid': 1234,
'title': 'Gee golly',
'description': 'This is fun.'
}]
result = filter.filter(result)
print result
assert result['items'] == [{'title': 'Gee golly', 'description': 'This is fun.'}]
def _generateArtifactList(options):
# load configuration
logging.info("Loading configuration...")
config = Configuration()
config.load(options)
# build list
logging.info("Building artifact list...")
listBuilder = ArtifactListBuilder(config)
artifactList = listBuilder.buildList()
logging.debug("Generated list contents:")
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
for classifier in artSpec.classifiers:
if classifier == "":
logging.debug(" %s:%s", gat, version)
else:
logging.debug(" %s:%s:%s", gat, classifier, version)
#filter list
logging.info("Filtering artifact list...")
listFilter = Filter(config)
artifactList = listFilter.filter(artifactList)
logging.debug("Filtered list contents:")
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
for classifier in artSpec.classifiers:
if classifier == "":
logging.debug(" %s:%s", gat, version)
else:
logging.debug(" %s:%s:%s", gat, classifier, version)
logging.info("Artifact list generation done")
return artifactList
class PushRedis(object):
connect_interval = 100
def __init__(self, pusher=None, kiss={}):
self.pusher = pusher
self.kiss = kiss
self.target = self.kiss["target"]
self.key_name = self.kiss["key_name"]
self.split_key = self.kiss["split_key"]
self.compile_key = self.kiss["compile_key"]
self.extend_info = self.kiss["extend_info"]
self.filt = Filter(self.target, self.compile_key)
self.tailfile = Tailfile(self.target, self.split_key)
def run(self):
for i in self.tailfile:
for j in self.filt.filter(i, self.extend_info):
self.pusher.rpush(self.key_name, j)
def _generateArtifactList(options, args):
config = Configuration()
if options.config or not args:
# load configuration
logging.info("Loading configuration...")
config.load(options)
else:
# create configuration
logging.info("Creating configuration...")
config.create(options, args)
# build list
logging.info("Building artifact list...")
listBuilder = ArtifactListBuilder(config)
artifactList = listBuilder.buildList()
logging.debug("Generated list contents:")
_logAL(artifactList)
#filter list
logging.info("Filtering artifact list...")
listFilter = Filter(config)
artifactList = listFilter.filter(artifactList)
logging.debug("Filtered list contents:")
_logAL(artifactList)
logging.info("Artifact list generation done")
if options.reportdir:
logging.info("Generating repository analysis report")
if hasattr(options, "reportname"):
reporter.generate_report(options.reportdir, config, artifactList,
options.reportname)
else:
reporter.generate_report(options.reportdir, config, artifactList,
None)
logging.info("Report has been generated")
return artifactList
def worker(id, msg, host):
url = msg['source'].strip()
print "looking up etag for", url
print host
headers = {'User-Agent': firefox_user_agent}
if url in host:
if "etag" in host[url]:
headers['If-None-Match'] = host[url]['etag']
if "last-modified" in host[url]:
headers['If-Modified-Since'] = host[url]['last-modified']
req = urllib2.Request(url, None, headers)
print "starting download"
try:
res = urllib2.urlopen(req)
print "completed download"
except urllib2.HTTPError as e:
print HTTPResponses[e.code]
save_job(_id=id, code=e.code)
return
except urllib2.URLError as e:
if e.reason[0][0] == -3:
print "request timed out"
error = {'code': e.reason[0][0], 'msg': e.reason[0][1]}
save_job(_id=id, error=error, code=500)
return
if 'parser' in msg and 'plugin_key' in msg['parser'] and msg['parser']['plugin_key'] in parsers:
parser = globals()[msg['parser']['plugin_key']]()
count, result = parser.parse(res.read())
filter = Filter(msg['processor']['config']['mappings'])
save_job(_id=id, count=count, result=filter.filter(result), code=200)
host[url] = dict(res.info())
print hosts.save(host)
def test_filter_run_out(self):
words = [
'cow',
'foo',
'fish',
'flamingo',
'trampoline',
'apollo',
]
res = [
'',
'',
'fish',
'flamingo',
'trampoline',
'apollo',
]
w = Filter(options)
filtered = []
for word in words:
filtered.append(w.filter(word))
self.assertEqual(filtered, res)
def _generateArtifactList(options, args):
config = Configuration()
if options.config or not args:
# load configuration
logging.info("Loading configuration...")
config.load(options)
else:
# create configuration
logging.info("Creating configuration...")
config.create(options, args)
# build list
logging.info("Building artifact list...")
listBuilder = ArtifactListBuilder(config)
artifactList = listBuilder.buildList()
logging.debug("Generated list contents:")
_logAL(artifactList)
#filter list
logging.info("Filtering artifact list...")
listFilter = Filter(config)
artifactList = listFilter.filter(artifactList, options.threadnum)
logging.debug("Filtered list contents:")
_logAL(artifactList)
logging.info("Artifact list generation done")
if options.reportdir:
logging.info("Generating repository analysis report")
if hasattr(options, "reportname"):
reporter.generate_report(options.reportdir, config, artifactList, options.reportname)
else:
reporter.generate_report(options.reportdir, config, artifactList, None)
logging.info("Report has been generated")
return artifactList
def test_filter_sudden_stop(self):
words = [
'cow',
'foo',
'fish',
'flamingo',
'enough',
'trampoline',
'apollo',
]
res = [
'',
'',
'fish',
'flamingo',
None,
'',
'',
]
filtered = []
w = Filter(options)
for word in words:
filtered.append(w.filter(word))
self.assertEqual(filtered, res)
from filter import Filter
from helpers.filter_math_helper import FilterMath
from signals_basic import Signal
FILTER_WINDOW = 30
MIN_ARRAY_VALUE = 1
MAX_ARRAY_VALUE = 100
LENGTH_ARRAY = 10000
if __name__ == '__main__':
signal_base = Signal(MIN_ARRAY_VALUE, MAX_ARRAY_VALUE, LENGTH_ARRAY)
number_array, x_volts = signal_base.generate_signal()
signal_base.show_signal(number_array, x_volts, 'исходный')
print(len(number_array))
x_volts = signal_base.generate_random(len(x_volts)) + x_volts
signal_base.show_signal(number_array, x_volts, 'с шумом')
print(len(x_volts))
filter_signal1 = Filter(FILTER_WINDOW, FilterMath.avg)
x_volts = filter_signal1.filter(x_volts) #ФИЛЬТР СКОЛЬЗЯЩЕГО СРЕДНЕГО
#filter_signal1 = Filter(FILTER_WINDOW, FilterMath.median)
#x_volts = filter_signal1.filter(x_volts) #МЕДИАННЫЙ ФИЛЬТР
print(len(x_volts))
signal_base.show_signal(number_array, x_volts, 'пропущенный через фильтр')
#x_volts = [filter_signal1.filter(point) for point in x_volts]
class Xycar(object):
def __init__(self, mode='start', hz=10):
self.rate = rospy.Rate(hz)
self.mode = mode
# ultrasonic
self.ultrasonic = None
self.filter = Filter()
self.sub_ultrasonic = rospy.Subscriber('xycar_ultrasonic',
Int32MultiArray,
self.callback_ultrasonic)
# image
self.img = None
self.bridge = CvBridge()
self.sub_img = rospy.Subscriber("/usb_cam/image_raw/", Image,
self.callback_img)
# qrcode
self.qr_dict = {
'1': 'ultrasonic',
'2': 'turn',
'3': 'dqn',
'4': 'bottle',
'5': 'yolo',
'6': 'pottedplant',
'7': 'command1',
'8': 'command2',
'9': 'park'
}
# motor
self.pub = rospy.Publisher('xycar_motor', xycar_motor, queue_size=1)
self.msg = xycar_motor()
# controllers
self.controller_ultrasonic = Ultrasonic()
self.controller_DQN = DQN()
self.controller_yolo = Yolo('bottle')
self.controller_ar = AR()
self.control_dict = {
'start': self.start,
'ultrasonic': self.ultrasonic_control,
'turn': self.turn_control,
'dqn': self.dqn_control,
'yolo': self.yolo_control,
'park': self.ar_control
}
def callback_ultrasonic(self, msg):
self.ultrasonic = self.filter.filter(msg.data)
def callback_img(self, msg):
self.img = self.bridge.imgmsg_to_cv2(msg, "bgr8")
def sleep(self):
self.rate.sleep()
def read(self, target):
if self.img is None:
return
qrcodes = pyzbar.decode(self.img)
if qrcodes:
text = qrcodes[0].data
if self.qr_dict[text[0]] == target:
self.mode = target
if target == 'turn':
self.controller_ultrasonic.backward = True
self.controller_ultrasonic.speed = 30
self.controller_ultrasonic.stop_counter_max = 30
elif target == 'yolo':
self.controller_yolo.detected = False
elif target == 'ar':
self.controller_ar.detected = False
def start(self):
self.read('ultrasonic')
return 0, 35
def ultrasonic_control(self):
self.read('turn')
return self.controller_ultrasonic.control(self.ultrasonic)
def turn_control(self):
self.read('dqn')
return self.controller_ultrasonic.control(self.ultrasonic)
def dqn_control(self):
self.read('yolo')
return self.controller_DQN.control(self.ultrasonic)
def yolo_control(self):
self.read('park')
if self.controller_yolo.detected:
return self.controller_yolo.control()
else:
return self.controller_ultrasonic.drive_go(self.ultrasonic)[0], 30
def ar_control(self):
if self.controller_ar.detected:
return self.controller_ar.control()
else:
return -10, 30
def control(self):
self.msg.angle, self.msg.speed = self.control_dict[self.mode]()
self.pub.publish(self.msg)
from filter import Filter
import sys
# Get arguments from user
train = sys.argv[1]
messages = sys.argv[2]
# Open files for reading and writing
train_file = open(train, "rb")
messages_file = open(messages, "rb")
predictions_file = open("predictions.txt", "w")
# Create new filter and train it using the train-file
f = Filter()
f.train(train_file)
#filter the messages in messages_file, write results to predictions_file
f.filter(messages_file, predictions_file)
# Close all the files
train_file.close()
messages_file.close()
predictions_file.close()
def submit2(self, window, variable):
global status
global path
global logfile
if path == "":
status.refresh("You didn't choose any video. Try again.")
logfile.write("[ERROR] Didn't choose any video.\n")
messagebox.showwarning(
title='warning',
message="You didn't choose any video. Try again.")
window.destroy()
else:
window.destroy()
status.refresh("Filtering trajectories data")
filter = Filter('trajectories.txt', 'trajectories_filtered.txt')
filterlength = askinteger("Input filter length",
"filter length: ",
initialvalue=60)
filter.filter(filterlength)
cap = cv2.VideoCapture(path)
ret, frame = cap.read()
mask = np.zeros_like(frame)
colors = []
for i in range(30):
color = (random.randint(0, 255), random.randint(0, 255),
random.randint(0, 255))
colors.append(color)
file = open('trajectories_filtered.txt', 'r')
trajs = []
for i, line in enumerate(file):
traj = Trajectory(i)
pts = line.split(',')[:-1]
for pt in pts:
xy = pt.split(' ')
x = float(xy[0])
y = float(xy[1])
xy = (x, y)
traj.addPoint(xy)
trajs.append(traj)
print("[INFO] Read in %d trajectories." % len(trajs))
status.refresh("Clustering")
clust = Clustering()
if variable.get() == 1:
logfile.write("[INFO] User used clusterAgglomerartive.\n")
num = askinteger("Input number of clusters",
"Number of clusters: ",
initialvalue=3)
res = clust.clusterAgglomerartive(trajs, num)
elif variable.get() == 2:
logfile.write("[INFO] User used spectrum cluster.\n")
res = clust.clusterSpectral(trajs)
status.refresh("Showing result")
print("printing image")
for traj in trajs:
for pt in traj.getPoints():
cv2.circle(mask, (int(pt[0]), int(pt[1])), 1,
colors[traj.getClusterIdx() - 1], -1)
count = [0] * res
for traj in trajs:
count[traj.getClusterIdx() - 1] += 1
print(count)
logfile.write("[INFO] Got %d clusters, %s.\n" % (res, str(count)))
cv2.imshow("result", cv2.add(frame, mask))
cv2.imwrite("after cluster.png", cv2.add(frame, mask))
if cv2.waitKey(0) == 27:
cap.release()
cv2.destroyAllWindows()
file.close()
