def test_filter_simple(self): res = [ '', '', '', 'cow', 'siamese', 'wonderland', 'foo', 'toothpaste', '', '', '', 'milky', 'flight-manual', 'toothpick', '', None, '', '', ] generated = [] filter = Filter(options) for word in words: generated.append(filter.filter(word)) self.assertEqual(generated, res)
class SearchHandler: def __init__(self): self.autoComplete = AutoComplete() self.resultFilter = Filter() self.googleCustom = GoogleCustomSearch() self.googleScholar = GoogleScholar() # returns a list of dictionaries for results def search(self, query: str, from_sites: tuple = None, ignore_sites: tuple = None) -> list: args = dict() results = list() if from_sites: args['from_sites'] = from_sites if ignore_sites: args['ignore_sites'] = ignore_sites results.append(self.googleCustom.search(query)) scholar_results = self.googleScholar.search(query) scholar_results.append('TRUSTED') results.append(scholar_results) filtered = self.resultFilter.filter(results, args) return filtered
def test_filter_finish_same_as_stop(self): words = [ 'cow', 'foo', 'fish', 'flamingo', 'trampoline', 'apollo', 'cow', 'pancakes', ] options = {'start': 'cow', 'stop': 'Flamingo', 'finish': 'flamingo'} res = [ '', 'foo', 'fish', None, '', '', '', 'pancakes', ] w = Filter(options) filtered = [] for word in words: filtered.append(w.filter(word)) self.assertEqual(res, filtered)
def _generateArtifactList(options): # load configuration logging.info("Loading configuration...") config = Configuration() config.load(options) # build list logging.info("Building artifact list...") listBuilder = ArtifactListBuilder(config) artifactList = listBuilder.buildList() logging.debug("Generated list contents:") for gat in artifactList: priorityList = artifactList[gat] for priority in priorityList: versionList = priorityList[priority] for version in versionList: logging.debug(" %s:%s", gat, version) #filter list logging.info("Filtering artifact list...") listFilter = Filter(config) artifactList = listFilter.filter(artifactList) logging.debug("Filtered list contents:") for gat in artifactList: priorityList = artifactList[gat] for priority in priorityList: versionList = priorityList[priority] for version in versionList: logging.debug(" %s:%s", gat, version) logging.info("Artifact list generation done") return artifactList
def do_stuff(train, messages): # Open files for reading and writing train_file = open(train, "rb") messages_file = open(messages, "rb") predictions_file = open("predictions.txt", "w") # Create new filter and train it using the train-file f = Filter() f.train(train_file) #filter the messages in messages_file, write results to predictions_file f.filter(messages_file, predictions_file) # Close all the files train_file.close() messages_file.close() predictions_file.close()
def test_filter(): mappings = [{'source': 'title'}, {'source': 'description'}, {'source': 'adfasdf'}, {'no source': 'adfasdf'}] filter = Filter(mappings) result = {} result['items'] = [{ 'author': 'Fred', 'guid': 1234, 'title': 'Gee golly', 'description': 'This is fun.' }] result = filter.filter(result) print result assert result['items'] == [{'title': 'Gee golly', 'description': 'This is fun.'}]
def _generateArtifactList(options): # load configuration logging.info("Loading configuration...") config = Configuration() config.load(options) # build list logging.info("Building artifact list...") listBuilder = ArtifactListBuilder(config) artifactList = listBuilder.buildList() logging.debug("Generated list contents:") for gat in artifactList: priorityList = artifactList[gat] for priority in priorityList: versionList = priorityList[priority] for version in versionList: artSpec = versionList[version] for classifier in artSpec.classifiers: if classifier == "": logging.debug(" %s:%s", gat, version) else: logging.debug(" %s:%s:%s", gat, classifier, version) #filter list logging.info("Filtering artifact list...") listFilter = Filter(config) artifactList = listFilter.filter(artifactList) logging.debug("Filtered list contents:") for gat in artifactList: priorityList = artifactList[gat] for priority in priorityList: versionList = priorityList[priority] for version in versionList: artSpec = versionList[version] for classifier in artSpec.classifiers: if classifier == "": logging.debug(" %s:%s", gat, version) else: logging.debug(" %s:%s:%s", gat, classifier, version) logging.info("Artifact list generation done") return artifactList
class PushRedis(object): connect_interval = 100 def __init__(self, pusher=None, kiss={}): self.pusher = pusher self.kiss = kiss self.target = self.kiss["target"] self.key_name = self.kiss["key_name"] self.split_key = self.kiss["split_key"] self.compile_key = self.kiss["compile_key"] self.extend_info = self.kiss["extend_info"] self.filt = Filter(self.target, self.compile_key) self.tailfile = Tailfile(self.target, self.split_key) def run(self): for i in self.tailfile: for j in self.filt.filter(i, self.extend_info): self.pusher.rpush(self.key_name, j)
def _generateArtifactList(options, args): config = Configuration() if options.config or not args: # load configuration logging.info("Loading configuration...") config.load(options) else: # create configuration logging.info("Creating configuration...") config.create(options, args) # build list logging.info("Building artifact list...") listBuilder = ArtifactListBuilder(config) artifactList = listBuilder.buildList() logging.debug("Generated list contents:") _logAL(artifactList) #filter list logging.info("Filtering artifact list...") listFilter = Filter(config) artifactList = listFilter.filter(artifactList) logging.debug("Filtered list contents:") _logAL(artifactList) logging.info("Artifact list generation done") if options.reportdir: logging.info("Generating repository analysis report") if hasattr(options, "reportname"): reporter.generate_report(options.reportdir, config, artifactList, options.reportname) else: reporter.generate_report(options.reportdir, config, artifactList, None) logging.info("Report has been generated") return artifactList
def worker(id, msg, host): url = msg['source'].strip() print "looking up etag for", url print host headers = {'User-Agent': firefox_user_agent} if url in host: if "etag" in host[url]: headers['If-None-Match'] = host[url]['etag'] if "last-modified" in host[url]: headers['If-Modified-Since'] = host[url]['last-modified'] req = urllib2.Request(url, None, headers) print "starting download" try: res = urllib2.urlopen(req) print "completed download" except urllib2.HTTPError as e: print HTTPResponses[e.code] save_job(_id=id, code=e.code) return except urllib2.URLError as e: if e.reason[0][0] == -3: print "request timed out" error = {'code': e.reason[0][0], 'msg': e.reason[0][1]} save_job(_id=id, error=error, code=500) return if 'parser' in msg and 'plugin_key' in msg['parser'] and msg['parser']['plugin_key'] in parsers: parser = globals()[msg['parser']['plugin_key']]() count, result = parser.parse(res.read()) filter = Filter(msg['processor']['config']['mappings']) save_job(_id=id, count=count, result=filter.filter(result), code=200) host[url] = dict(res.info()) print hosts.save(host)
def test_filter_run_out(self): words = [ 'cow', 'foo', 'fish', 'flamingo', 'trampoline', 'apollo', ] res = [ '', '', 'fish', 'flamingo', 'trampoline', 'apollo', ] w = Filter(options) filtered = [] for word in words: filtered.append(w.filter(word)) self.assertEqual(filtered, res)
def _generateArtifactList(options, args): config = Configuration() if options.config or not args: # load configuration logging.info("Loading configuration...") config.load(options) else: # create configuration logging.info("Creating configuration...") config.create(options, args) # build list logging.info("Building artifact list...") listBuilder = ArtifactListBuilder(config) artifactList = listBuilder.buildList() logging.debug("Generated list contents:") _logAL(artifactList) #filter list logging.info("Filtering artifact list...") listFilter = Filter(config) artifactList = listFilter.filter(artifactList, options.threadnum) logging.debug("Filtered list contents:") _logAL(artifactList) logging.info("Artifact list generation done") if options.reportdir: logging.info("Generating repository analysis report") if hasattr(options, "reportname"): reporter.generate_report(options.reportdir, config, artifactList, options.reportname) else: reporter.generate_report(options.reportdir, config, artifactList, None) logging.info("Report has been generated") return artifactList
def test_filter_sudden_stop(self): words = [ 'cow', 'foo', 'fish', 'flamingo', 'enough', 'trampoline', 'apollo', ] res = [ '', '', 'fish', 'flamingo', None, '', '', ] filtered = [] w = Filter(options) for word in words: filtered.append(w.filter(word)) self.assertEqual(filtered, res)
from filter import Filter from helpers.filter_math_helper import FilterMath from signals_basic import Signal FILTER_WINDOW = 30 MIN_ARRAY_VALUE = 1 MAX_ARRAY_VALUE = 100 LENGTH_ARRAY = 10000 if __name__ == '__main__': signal_base = Signal(MIN_ARRAY_VALUE, MAX_ARRAY_VALUE, LENGTH_ARRAY) number_array, x_volts = signal_base.generate_signal() signal_base.show_signal(number_array, x_volts, 'исходный') print(len(number_array)) x_volts = signal_base.generate_random(len(x_volts)) + x_volts signal_base.show_signal(number_array, x_volts, 'с шумом') print(len(x_volts)) filter_signal1 = Filter(FILTER_WINDOW, FilterMath.avg) x_volts = filter_signal1.filter(x_volts) #ФИЛЬТР СКОЛЬЗЯЩЕГО СРЕДНЕГО #filter_signal1 = Filter(FILTER_WINDOW, FilterMath.median) #x_volts = filter_signal1.filter(x_volts) #МЕДИАННЫЙ ФИЛЬТР print(len(x_volts)) signal_base.show_signal(number_array, x_volts, 'пропущенный через фильтр') #x_volts = [filter_signal1.filter(point) for point in x_volts]
class Xycar(object): def __init__(self, mode='start', hz=10): self.rate = rospy.Rate(hz) self.mode = mode # ultrasonic self.ultrasonic = None self.filter = Filter() self.sub_ultrasonic = rospy.Subscriber('xycar_ultrasonic', Int32MultiArray, self.callback_ultrasonic) # image self.img = None self.bridge = CvBridge() self.sub_img = rospy.Subscriber("/usb_cam/image_raw/", Image, self.callback_img) # qrcode self.qr_dict = { '1': 'ultrasonic', '2': 'turn', '3': 'dqn', '4': 'bottle', '5': 'yolo', '6': 'pottedplant', '7': 'command1', '8': 'command2', '9': 'park' } # motor self.pub = rospy.Publisher('xycar_motor', xycar_motor, queue_size=1) self.msg = xycar_motor() # controllers self.controller_ultrasonic = Ultrasonic() self.controller_DQN = DQN() self.controller_yolo = Yolo('bottle') self.controller_ar = AR() self.control_dict = { 'start': self.start, 'ultrasonic': self.ultrasonic_control, 'turn': self.turn_control, 'dqn': self.dqn_control, 'yolo': self.yolo_control, 'park': self.ar_control } def callback_ultrasonic(self, msg): self.ultrasonic = self.filter.filter(msg.data) def callback_img(self, msg): self.img = self.bridge.imgmsg_to_cv2(msg, "bgr8") def sleep(self): self.rate.sleep() def read(self, target): if self.img is None: return qrcodes = pyzbar.decode(self.img) if qrcodes: text = qrcodes[0].data if self.qr_dict[text[0]] == target: self.mode = target if target == 'turn': self.controller_ultrasonic.backward = True self.controller_ultrasonic.speed = 30 self.controller_ultrasonic.stop_counter_max = 30 elif target == 'yolo': self.controller_yolo.detected = False elif target == 'ar': self.controller_ar.detected = False def start(self): self.read('ultrasonic') return 0, 35 def ultrasonic_control(self): self.read('turn') return self.controller_ultrasonic.control(self.ultrasonic) def turn_control(self): self.read('dqn') return self.controller_ultrasonic.control(self.ultrasonic) def dqn_control(self): self.read('yolo') return self.controller_DQN.control(self.ultrasonic) def yolo_control(self): self.read('park') if self.controller_yolo.detected: return self.controller_yolo.control() else: return self.controller_ultrasonic.drive_go(self.ultrasonic)[0], 30 def ar_control(self): if self.controller_ar.detected: return self.controller_ar.control() else: return -10, 30 def control(self): self.msg.angle, self.msg.speed = self.control_dict[self.mode]() self.pub.publish(self.msg)
from filter import Filter import sys # Get arguments from user train = sys.argv[1] messages = sys.argv[2] # Open files for reading and writing train_file = open(train, "rb") messages_file = open(messages, "rb") predictions_file = open("predictions.txt", "w") # Create new filter and train it using the train-file f = Filter() f.train(train_file) #filter the messages in messages_file, write results to predictions_file f.filter(messages_file, predictions_file) # Close all the files train_file.close() messages_file.close() predictions_file.close()
def submit2(self, window, variable): global status global path global logfile if path == "": status.refresh("You didn't choose any video. Try again.") logfile.write("[ERROR] Didn't choose any video.\n") messagebox.showwarning( title='warning', message="You didn't choose any video. Try again.") window.destroy() else: window.destroy() status.refresh("Filtering trajectories data") filter = Filter('trajectories.txt', 'trajectories_filtered.txt') filterlength = askinteger("Input filter length", "filter length: ", initialvalue=60) filter.filter(filterlength) cap = cv2.VideoCapture(path) ret, frame = cap.read() mask = np.zeros_like(frame) colors = [] for i in range(30): color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)) colors.append(color) file = open('trajectories_filtered.txt', 'r') trajs = [] for i, line in enumerate(file): traj = Trajectory(i) pts = line.split(',')[:-1] for pt in pts: xy = pt.split(' ') x = float(xy[0]) y = float(xy[1]) xy = (x, y) traj.addPoint(xy) trajs.append(traj) print("[INFO] Read in %d trajectories." % len(trajs)) status.refresh("Clustering") clust = Clustering() if variable.get() == 1: logfile.write("[INFO] User used clusterAgglomerartive.\n") num = askinteger("Input number of clusters", "Number of clusters: ", initialvalue=3) res = clust.clusterAgglomerartive(trajs, num) elif variable.get() == 2: logfile.write("[INFO] User used spectrum cluster.\n") res = clust.clusterSpectral(trajs) status.refresh("Showing result") print("printing image") for traj in trajs: for pt in traj.getPoints(): cv2.circle(mask, (int(pt[0]), int(pt[1])), 1, colors[traj.getClusterIdx() - 1], -1) count = [0] * res for traj in trajs: count[traj.getClusterIdx() - 1] += 1 print(count) logfile.write("[INFO] Got %d clusters, %s.\n" % (res, str(count))) cv2.imshow("result", cv2.add(frame, mask)) cv2.imwrite("after cluster.png", cv2.add(frame, mask)) if cv2.waitKey(0) == 27: cap.release() cv2.destroyAllWindows() file.close()