def __init__(self, api_host, gatsby_host, alias_host, num_threads): self.select_clause = "mpidStr AS \'mpid\', priceRange, aggregatedRatings, modelTitle AS \'title\', brandName, categoryNamePath, searchScore, brandName, storeId, image" self.page_size = 500 self.country_code = 356 self.gatsby_query = ProductGatsbyQuery(self.select_clause, self.page_size, self.country_code, "/products/search2", gatsby_host) self.gatsbyPB_query = ProductGatsbyQuery(self.select_clause, self.page_size, self.country_code, "/products/search2", gatsby_host) self.api_query = ProductApiQuery("http", api_host, "/v2.1/search", "IN", 50) self.alias_service = DataCollector( ProductAliasQuery("http", alias_host, "/search", "IN")) self.ranking_model = RankingModel() self.gatsby_queries = Queue(2) self.gatsbyPB_queries = Queue(2) self.api_queries = Queue(2) worker = Worker(DataCollector(self.gatsby_query), self.ranking_model, self.gatsby_queries) # worker.daemon = True print "currently running thread:\t", worker.getName() worker.start() worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model, self.gatsbyPB_queries) print "currently running thread:\t", worker.getName() # worker.daemon = True worker.start() # for thread in range(2): # worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model, self.gatsbyPB_queries) # print "currently running thread:\t", worker.getName() # worker.daemon = True # worker.start() # # for thread in range(2): # worker = Worker(DataCollector(self.gatsby_query), self.ranking_model, self.gatsby_queries) # worker.daemon = True # worker.start() worker = Worker(DataCollector(self.api_query), self.ranking_model, self.api_queries) print "currently running thread:\t", worker.getName() # worker.daemon = True worker.start()
def run_random_search(verbose, num_diff_experiments, num_repeat_experiment, allow_duplicates=False, df_path=None, overwrite=True, data_to_collect=POSSIBLE_DATA, MVP_key='waitingTime', save_model=True): grid = load_constants('constants/constants-grid.json') if not allow_duplicates: _, num_choices = get_num_grid_choices(grid) num_diff_experiments = min(num_choices, num_diff_experiments) # Make grid choice generator grid_choice_gen = grid_choices_random(grid, num_diff_experiments) for diff_experiment, constants in enumerate(grid_choice_gen): data_collector_obj = DataCollector( data_to_collect, MVP_key, constants, 'test' if constants['agent']['agent_type'] == 'rule' else 'eval', df_path, overwrite if diff_experiment == 0 else False, verbose) for same_experiment in range(num_repeat_experiment): print(' --- Running experiment {}.{} / {}.{} --- '.format( diff_experiment + 1, same_experiment + 1, num_diff_experiments, num_repeat_experiment)) if save_model: data_collector_obj.set_save_model_path( 'models/saved_models/random_{}-{}.pt'.format( diff_experiment + 1, same_experiment + 1)) run_experiment(diff_experiment + 1, same_experiment + 1, constants, data_collector_obj)
def register(config, group_id=None): """ Do registration using basic auth """ username = config.get(APP_NAME, 'username') password = config.get(APP_NAME, 'password') if (((username == "") and (password == "") and (config.get(APP_NAME, 'authmethod') == 'BASIC')) and not (config.get(APP_NAME, 'auto_config'))): # Get input from user print "Please enter your Red Hat Customer Portal Credentials" sys.stdout.write('User Name: ') username = raw_input().strip() password = getpass.getpass() sys.stdout.write("Would you like to save these credentials? (y/n) ") save = raw_input().strip() config.set(APP_NAME, 'username', username) config.set(APP_NAME, 'password', password) logger.debug("savestr: %s", save) if save.lower() == "y" or save.lower() == "yes": logger.debug("writing user/pass to config file") cmd = ("/bin/sed -e 's/^username.*=.*$/username="******"/' " + "-e 's/^password.*=.*$/password="******"/' " + constants.default_conf_file) status = DataCollector().run_command_get_output(cmd, nolog=True) config_file = open(constants.default_conf_file, 'w') config_file.write(status['output']) config_file.flush() pconn = InsightsConnection(config) return pconn.register(group_id)
def __init__(self, data_collector=None): if data_collector is None: data_collector = DataCollector() else: self.data_collector = data_collector self.bigrams = dict() self.get_unique_bigrams()
def testRunWithError(self): collector = DataCollector(in_path=IN_FILE_BAD, ot_path_data=OT_FILE_DATA, ot_path_doc=OT_FILE_DOC) collector.run() df = pd.read_csv(OT_FILE_DATA) self.assertEqual(len(df["Biomodel_Id"]), 3)
def setup_data_threads(self, sample_time = None, decimation = None, start_threads = False): if self.is_slave: if not self.datastream_queue: self.datastream_queue = [] # TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller. if not self.datastream_poller: self.datastream_poller = DataPoller(self.datastream_queue, decimation) # TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati if not self.datastream_collector: # TODO: Fermare i thread alla chiusura del target self.datastream_collector = DataCollector(self.config.getAttr('address'), self.config.getAttr('slave_data_port'), self.datastream_queue) if start_threads: print("Starting data threads...") if sample_time: self.datastream_poller.updateSampleTime(sample_time) self.datastream_poller.start() self.datastream_collector.start() print("Data threads started...") else: print("Can't start data threads without a sample time") return False return True else: print("Can't setup the data stream threads on a not-slave server") return False
def __init__(self): with DataCollector() as collector: result_channels = collector.get_telemetry_outputs() with DataServer(result_channels) as transmitter: print("Press ctrl+c to exit") signal.signal(signal.SIGINT, self.exit_handler) # TODO signal.pause() is not available on Windows systems signal.pause()
def mean_std_of_states(): env = MountainCarWithResetEnv() samples_to_collect = 100000 states, actions, rewards, next_states, done_flags = DataCollector(env).collect_data(samples_to_collect) all_states = np.concatenate((states, next_states)) states_mean = np.mean(all_states, axis=0) states_std = np.std(all_states, axis=0) print("states_mean: {}, states_std: {}".format(states_mean, states_std))
def main(): args = get_args() data_collector_obj = DataCollector(args.debug) if args.debug: print("Got arguments " + str(args.years) + ", " + str(args.players) + ", " + args.mode + ", " + str(args.singles)) data_collector_obj.scrape_last_n_years_of_k_players( args.years, args.players, args.mode, "singles" if args.singles else "doubles")
def training_the_model(samples_to_collect=100000, seed=100): number_of_kernels_per_dim = [10, 8] gamma = 0.999 w_updates = 20 evaluation_number_of_games = 50 evaluation_max_steps_per_game = 300 np.random.seed(seed) env = MountainCarWithResetEnv() # collect data states, actions, rewards, next_states, done_flags = DataCollector( env).collect_data(samples_to_collect) # get data success rate data_success_rate = np.sum(rewards) / len(rewards) print(f'Data Success Rate {data_success_rate}') # standardize data data_transformer = DataTransformer() data_transformer.set_using_states( np.concatenate((states, next_states), axis=0)) states = data_transformer.transform_states(states) next_states = data_transformer.transform_states(next_states) # process with radial basis functions feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim) # encode all states: encoded_states = feature_extractor.encode_states_with_radial_basis_functions( states) encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions( next_states) # set a new linear policy linear_policy = LinearPolicy(feature_extractor.get_number_of_features(), 3, True) # but set the weights as random linear_policy.set_w(np.random.uniform(size=linear_policy.w.shape)) # start an object that evaluates the success rate over time evaluator = GamePlayer(env, data_transformer, feature_extractor, linear_policy) success_rate_vs_iteration = list() for lspi_iteration in range(w_updates): print(f'Starting LSPI iteration {lspi_iteration}') new_w = compute_lspi_iteration(encoded_states, encoded_next_states, actions, rewards, done_flags, linear_policy, gamma) norm_diff = linear_policy.set_w(new_w) success_rate = evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game) success_rate_vs_iteration.append(success_rate) if norm_diff < 0.00001: break print('LSPI Done') return success_rate_vs_iteration
def setUp(self): data_collector = DataCollector(config=config) self.data_after = data_collector.data_collector() path = 'devlab/tests' file_name = config.rollback_params['data_file_names']['PRE'] pre_file_path = os.path.join(data_collector.main_folder, path, file_name) with open(pre_file_path, "r") as f: self.pre_data = yaml.load(f)
def perfrom_experiment(self, experiment_name='test', movement_list=[]): # 1. We save the background image: dc = DataCollector(only_one_shot=False, automatic=True) dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=False, save=True, directory=experiment_name+'/air', iteration=-1) print "Air data gathered" # 2. We perfomr the experiment: i = 0 if not os.path.exists(experiment_name): # If the directory does not exist, we create it os.makedirs(experiment_name) for movement in movement_list: path = experiment_name + '/p_' + str(i) + '/' self.palpate(speed=40, force_list=self.force_list, save=True, path=path) self.move_cart_mm(movement[0], movement[1], movement[2]) time.sleep(6) i += 1 path = experiment_name + '/p_' + str(i) + '/' self.palpate(speed=40, force_list=self.force_list, save=True, path=path)
def palpate(self, speed=40, force_list=[1, 10, 20, 40], save=False, path=''): # 0. We create the directory if save is True and not os.path.exists(path): # If the directory does not exist, we create it os.makedirs(path) # 1. We get and save the cartesian coord. dc = DataCollector(only_one_shot=False, automatic=True) cart = dc.getCart() if save is True: np.save(path + '/cart.npy', cart) # 2. We get wsg forces and gs images at every set force and store them i = 0 for force in force_list: self.close_gripper_f(grasp_speed=speed, grasp_force=force) print "Applying: " + str(force) time.sleep(1.0) dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=True, save=save, directory=path, iteration=i) self.open_gripper() time.sleep(1.0) i += 1
def __init__(self): self.screen = MeteoScreen() self.collector = DataCollector() self.data = InMemorySensorDataLogger() self.collector.add_logger(self.data) file_logger = ToFileSensorDataLogger('./db') self.collector.add_logger(file_logger) now = datetime.datetime.now() last24h_data = file_logger.load_data(now - datetime.timedelta(days=1), now) self.data.set_data(last24h_data) self.screen_update_interval = 10 self.current_screen_draw = self.draw_screen1 self.update_timer_stop = threading.Event() self.screen_update_lock = threading.RLock() def timer(): while not self.update_timer_stop.is_set(): time.sleep(self.screen_update_interval) self.update_screen() self.timer_thread = threading.Thread(target=timer)
def run_normal(verbose, num_experiments=1, df_path=None, overwrite=True, data_to_collect=POSSIBLE_DATA, MVP_key='waitingTime', save_model=True, load_model_file=None): # if loading, then dont save if load_model_file: save_model = False if not df_path: df_path = 'run-data.xlsx' # def. path # Load constants constants = load_constants('constants/constants.json') data_collector_obj = DataCollector( data_to_collect, MVP_key, constants, 'test' if constants['agent']['agent_type'] == 'rule' or load_model_file else 'eval', df_path, overwrite, verbose) loaded_model = None if load_model_file: loaded_model = torch.load('models/saved_models/' + load_model_file) for exp in range(num_experiments): print(' --- Running experiment {} / {} --- '.format( exp + 1, num_experiments)) if save_model: data_collector_obj.set_save_model_path( 'models/saved_models/normal_{}.pt'.format(exp + 1)) run_experiment(exp + 1, None, constants, data_collector_obj, loaded_model=loaded_model)
def run(self): """Main loop """ self.startCapture() data_collector = DataCollector(self.dataset) modeFullFace = 0 modeOneEye = 1 modePictures = 2 modeTwoEyes = 3 modeImgDB = 4 modeDemo = 5 mode = modeDemo keepLoop = True current_t = time.clock() previous_t = current_t while keepLoop: pressed_key = cv2.waitKey(1) current_t = time.clock() #print('\nclock : ', current_t - previous_t) previous_t = current_t img = self.getCameraImage() if(mode == modeOneEye): ex = 300 ey = 50 eh = 200 ew = 200 face = Face(img.frame, img.canvas, 0, 0, 640, 480) eye = Eye(face.frame, face.canvas, ex, ey, ew, eh) eye.draw(face) eye.iris.normalizeIris() elif(mode == modeTwoEyes): face = Face(img.frame, img.canvas, 0, 0, 640, 480) left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT) left_eye.draw(face) left_eye.iris.normalizeIris() right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT) right_eye.draw(face) right_eye.iris.normalizeIris() elif(mode == modeFullFace): face, left_eye, right_eye = img.detectEyes(self.bufferFace, self.bufferLeftEye, self.bufferRightEye) if face: face.draw(img) if left_eye: left_eye.draw(face) left_eye.iris.normalizeIris() if right_eye: right_eye.draw(face) right_eye.iris.normalizeIris() elif(mode == modeImgDB): img_db = ImageDB("./IR_Database/MMU Iris Database/") print(img_db.estimateUser(img_db.bits[0])) exit() elif(mode == modeDemo): path = "./TB_Database/" face = Face(img.frame, img.canvas, 0, 0, 640, 480) left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT) left_eye.draw(face) left_eye.iris.normalizeIris() right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT) right_eye.draw(face) right_eye.iris.normalizeIris() if(ord('0') <= pressed_key & 0xFF <= ord('9')): print('0-9') id_person = chr(pressed_key & 0xFF) while((pressed_key & 0xFF) not in [ord('a'), ord('p')]): pressed_key = cv2.waitKey(50) print(pressed_key & 0xFF) if(pressed_key & 0xFF == ord('a')): cv2.imwrite(path + id_person + '/left/' + str(int(time.time() * 1000)) + '.bmp', left_eye.frame) cv2.imwrite(path + id_person + '/right/' + str(int(time.time() * 1000)) + '.bmp', right_eye.frame) else: path = "./IR_Database/MMU Iris Database/" for dir in os.listdir(path): subpath = path + dir + '/' if(os.path.isdir(subpath)): print(dir) for subdir in os.listdir(subpath): subsubpath = subpath + subdir + '/' if(os.path.isdir(subsubpath)): print('\t', subdir) for fname in os.listdir(subsubpath): fpath = subsubpath + fname if(os.path.isfile(fpath) and os.path.splitext(fname)[1] == '.bmp'): print('\t\t', fname) img = Image(cv2.imread(fpath)) face = Face(img.frame, img.canvas) eye = Eye(face.frame, face.canvas, padding=10) eye.draw(face) eye.iris.normalizeIris() img.show() pressed_key = cv2.waitKey(1000) exit() # Controls if pressed_key & 0xFF == ord('q'): keepLoop = False #if pressed_key & 0xFF == ord('s'): # self.dataset.save() #if pressed_key & 0xFF == ord('l'): # self.dataset.load() #if pressed_key & 0xFF == ord('m'): # self.showMoments = not self.showMoments #if pressed_key & 0xFF == ord('e'): # self.showEvaluation = not self.showEvaluation #data_collector.step(img.canvas, pressed_key, left_eye, right_eye) #txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset)) #cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) #cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) #if left_eye and right_eye: # direction = self.dataset.estimateDirection(left_eye.computeMomentVectors(), right_eye.computeMomentVectors()) # txt = 'Estimated direction: {}'.format(direction.name) # cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) # cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) img.show() #if self.showEvaluation: # fig = self.dataset.showValidationScoreEvolution() # plt.show() # self.showEvaluation = False #if self.showMoments: # fig = self.dataset.drawVectorizedMoments() # plt.show() # # cv2.imshow('moments', self.fig2cv(fig)) # # plt.close(fig) # self.showMoments = False self.stopCapture()
if __name__ == '__main__': samples_to_collect = 100000 # samples_to_collect = 150000 # samples_to_collect = 10000 number_of_kernels_per_dim = [10, 8] gamma = 0.99 w_updates = 100 evaluation_number_of_games = 10 evaluation_max_steps_per_game = 1000 np.random.seed(123) # np.random.seed(234) env = MountainCarWithResetEnv() # collect data states, actions, rewards, next_states, done_flags = DataCollector( env).collect_data(samples_to_collect) # get data success rate data_success_rate = np.sum(rewards) / len(rewards) print(f'success rate {data_success_rate}') # standardize data data_transformer = DataTransformer() data_transformer.set_using_states( np.concatenate((states, next_states), axis=0)) states = data_transformer.transform_states(states) next_states = data_transformer.transform_states(next_states) # process with radial basis functions feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim) # encode all states: encoded_states = feature_extractor.encode_states_with_radial_basis_functions( states) encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions(
def __init__(self, earthquake_data, earthquake_data_clean): self.earthquake_data = earthquake_data self.earthquake_data_clean = earthquake_data_clean self.data_collector = DataCollector() self.new_location = DataCollector()
def setUp(self): self.collector = DataCollector(in_path=IN_FILE, ot_path_data=OT_FILE_DATA, ot_path_doc=OT_FILE_DOC)
def run_lspi(seed, w_updates=20, samples_to_collect=100000, evaluation_number_of_games=1, evaluation_max_steps_per_game=200, thresh=0.00001, only_final=False): """ This is the main lspi function :param seed: random seed for the run :param w_updates: how many w updates to do :param samples_to_collect: how many samples to collect :param evaluation_number_of_games: how many game evaluations to do :param evaluation_max_steps_per_game: how many steps to allow the evaluation game to run :param thresh: the threshold for the stopping condition :param only_final: run evaluation only at the end of the run :return: None """ res_dir = './Results/' np.random.seed(seed) number_of_kernels_per_dim = [12, 10] gamma = 0.999 env = MountainCarWithResetEnv() # collect data states, actions, rewards, next_states, done_flags = DataCollector( env).collect_data(samples_to_collect) # get data success rate data_success_rate = np.sum(rewards) / len(rewards) print('success rate: {}'.format(data_success_rate)) # standardize data data_transformer = DataTransformer() data_transformer.set_using_states( np.concatenate((states, next_states), axis=0)) states = data_transformer.transform_states(states) next_states = data_transformer.transform_states(next_states) # process with radial basis functions feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim) # encode all states: encoded_states = feature_extractor.encode_states_with_radial_basis_functions( states) encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions( next_states) # set a new linear policy linear_policy = LinearPolicy(feature_extractor.get_number_of_features(), 3, True) # but set the weights as random linear_policy.set_w(np.random.uniform(size=linear_policy.w.shape)) # start an object that evaluates the success rate over time evaluator = GamePlayer(env, data_transformer, feature_extractor, linear_policy) # success_rate = evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game) # print("Initial success rate: {}".format(success_rate)) performances = [] if not only_final: performances.append( evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game)) read = False if read: with open(res_dir + 'weight.pickle', 'rb') as handle: new_w = pickle.load(handle) linear_policy.set_w(np.expand_dims(new_w, 1)) for lspi_iteration in range(w_updates): print('starting lspi iteration {}'.format(lspi_iteration)) new_w = compute_lspi_iteration(encoded_states, encoded_next_states, actions, rewards, done_flags, linear_policy, gamma) with open(res_dir + 'weight.pickle', 'wb') as handle: pickle.dump(new_w, handle, protocol=pickle.HIGHEST_PROTOCOL) norm_diff = linear_policy.set_w(new_w) if not only_final: performances.append( evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game)) if norm_diff < thresh: break print('done lspi') if not only_final: with open(res_dir + 'perf' + str(seed) + '.pickle', 'wb') as handle: pickle.dump(performances, handle, protocol=pickle.HIGHEST_PROTOCOL) if only_final: score = evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game) with open(res_dir + 'final_perf' + str(samples_to_collect) + '.pickle', 'wb') as handle: pickle.dump(score, handle, protocol=pickle.HIGHEST_PROTOCOL) evaluator.play_game(evaluation_max_steps_per_game, render=True)
def collect_data_and_upload(config, options): """ All the heavy lifting done here """ pconn = InsightsConnection(config) try: branch_info = pconn.branch_info() except requests.ConnectionError: logger.error("ERROR: Could not connect to determine branch information") sys.exit() except LookupError: logger.error("ERROR: Could not determine branch information") sys.exit() pc = InsightsConfig(config, pconn) dc = DataCollector() start = time.clock() collection_rules, rm_conf = pc.get_conf(options.update) elapsed = (time.clock() - start) logger.debug("Collection Rules Elapsed Time: %s", elapsed) start = time.clock() logger.info('Starting to collect Insights data') dc.run_commands(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") if not options.no_tar_file: tar_file = dc.done(config, rm_conf) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') for tries in range(options.retries): status = pconn.upload_archive(tar_file) if status == 201: logger.info("Upload completed successfully!") break else: logger.error("Upload attempt %d of %d failed! Status Code: %s", tries+1, options.retries, status) if tries +1 != options.retries: logger.info("Waiting %d seconds then retrying", constants.sleep_time) time.sleep(constants.sleep_time) else: logger.error("All attempts to upload have failed!") logger.error("Please see %s for additional information", constants.default_log_file) if not obfuscate and not options.keep_archive: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: logger.info('See Insights data in %s', tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir)
pos = tuple(frame.position[:3]) rot = tfx.tb_angles(frame.rotation) rot = (rot.yaw_deg, rot.pitch_deg, rot.roll_deg) pickle.dump({'pos': pos, 'rot': rot, 'estimate': estimate}, f) f.close() psm1 = robot("PSM1") psm1.open_gripper(80) time.sleep(2) d = DataCollector() time.sleep(2) img = cv2.medianBlur(d.left_image[:, 850:], 13) img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.bilateralFilter(img, 11, 17, 17) output = cv2.Canny(img, 100, 200) (cnts, _) = cv2.findContours(output.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cnts = sorted(cnts, key=cv2.contourArea, reverse=True) for c in cnts: # approximate the contour peri = cv2.arcLength(c, True)
new_SD_list, new_not_SD = loop_through_ids( self.ids_social_distance) assert len(new_SD_list) == 0 # Switch people for id in new_SD: self.ids_not_social_distance.remove(id) self.ids_social_distance.add(id) for id in new_not_SD: self.ids_social_distance.remove(id) self.ids_not_social_distance.add(id) if render: pygame.display.flip() screen.fill((0, 0, 0)) # Frames per second if self.render_C['fps']: clock.tick(self.render_C['fps']) self.data_collect.reset(t + 1, last=True) if __name__ == '__main__': constants = json.load(open('constants.json')) # Can save a run as an experiment which saves the data, visualizations and constants in a experiments directory data_collect = DataCollector(constants, save_experiment=True, print_visualizations=True) # Can print data (look at `data_options` at top of `data_collector.py` for options) and how often to print data_collect.set_print_options(basic_to_print=['S', 'I', 'R', 'death'], frequency=1) CA = CellularAutomation(constants, data_collect) # Can render each timestep with pygame CA.run(render=True)
from crawler import Crawler from schemas import Page from data_collector import DataCollector import validators dc = DataCollector('localhost', 27017, '20171013-nigth') readLinks = [] class Utils: invalidWords = [ 'facebook', 'fb', 'google', 'instagram', 'twitter', 'youtube', 'bing', 'yahoo', 'ecosia', '.pdf', '.doc', '.xls', '.png', '.jpg', '.gif', 'ads', 'goo.gl', 'microsoft', 'gmail', 'skype', 'download', 'apple', 'pinterest', '.ppt', 'mozilla', 'microsoft', 'ebay', 'amazon', 'newegg', 'soundcloud', 'advertise', 'advertising' ] @staticmethod def isValidURL(url): containsInvalidWord = False try: url = str(url) for word in Utils.invalidWords: if url.find(word) > -1: containsInvalidWord = True break if validators.url(url) and containsInvalidWord == False:
def __init__(self): self.data_collector = DataCollector() self.new_location = DataCollector()
def setup_class(cls): print("Setting up CLASS {0}".format(cls.__name__)) cls.d = DataCollector()
def __init__(self): self.di = DataCollector()
def run(self): """Main loop """ self.startCapture() data_collector = DataCollector(self.dataset) keepLoop = True while keepLoop: pressed_key = cv2.waitKey(1) img = self.getCameraImage() face, left_eye, right_eye = img.detectEyes(self.bufferFace, self.bufferLeftEye, self.bufferRightEye) if face: face.draw(img) if left_eye: left_eye.draw(face) if right_eye: right_eye.draw(face) # Controls if pressed_key & 0xFF == ord('q'): keepLoop = False if pressed_key & 0xFF == ord('s'): self.dataset.save() if pressed_key & 0xFF == ord('l'): self.dataset.load() if pressed_key & 0xFF == ord('m'): self.showMoments = not self.showMoments if pressed_key & 0xFF == ord('e'): self.showEvaluation = not self.showEvaluation data_collector.step(img.canvas, pressed_key, left_eye, right_eye) txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset)) cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) if left_eye and right_eye: direction = self.dataset.estimateDirection( left_eye.computeMomentVectors(), right_eye.computeMomentVectors()) txt = 'Estimated direction: {}'.format(direction.name) cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) img.show() if self.showEvaluation: fig = self.dataset.showValidationScoreEvolution() plt.show() self.showEvaluation = False if self.showMoments: fig = self.dataset.drawVectorizedMoments() plt.show() # cv2.imshow('moments', self.fig2cv(fig)) # plt.close(fig) self.showMoments = False self.stopCapture()
parser.add_option("-k", action="store", dest="k", type="long", default=3) parser.add_option("--temp", action="store", dest="temp_rate", type="float", default=0.9) parser.add_option("--iters", action="store", dest="iters", type="float", default=100) parser.add_option("--max_t", action="store", dest="max_temp", type="float", default=100) parser.add_option("--min_t", action="store", dest="min_temp", type="float", default=0.5) parser.add_option("--hh", action="store", type="string", dest="hh") global options (options, _) = parser.parse_args() # data collector configuration global collector collector = DataCollector()
def collect_data_and_upload(rc=0): """ All the heavy lifting done here Run through "targets" - could be just ONE (host, default) or ONE (container/image) """ # initialize collection targets # for now we do either containers OR host -- not both at same time if InsightsClient.options.container_mode: logger.debug("Client running in container/image mode.") logger.debug("Scanning for matching container/image.") targets = get_targets() else: logger.debug("Host selected as scanning target.") targets = constants.default_target # if there are no targets to scan then bail if not len(targets): logger.debug("No targets were found. Exiting.") sys.exit(1) if InsightsClient.options.offline: logger.warning("Assuming remote branch and leaf value of -1") pconn = None branch_info = constants.default_branch_info else: pconn = InsightsConnection() # TODO: change these err msgs to be more meaningful , i.e. # "could not determine login information" if pconn: try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information") except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information") pc = InsightsConfig(pconn) tar_file = None if InsightsClient.options.just_upload: if not os.path.exists(InsightsClient.options.just_upload): logger.error('No file %s', InsightsClient.options.just_upload) return 1 tar_file = InsightsClient.options.just_upload rc = _do_upload(pconn, tar_file, 'dummy', 0) return rc # load config from stdin/file if specified try: stdin_config = {} if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) elif InsightsClient.options.from_stdin: stdin_config = json.load(sys.stdin) if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and ('uploader.json' not in stdin_config or 'sig' not in stdin_config)): raise ValueError if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and 'branch_info' in stdin_config and stdin_config['branch_info'] is not None): branch_info = stdin_config['branch_info'] except: logger.error('ERROR: Invalid config for %s! Exiting...', ('--from-file' if InsightsClient.options.from_file else '--from-stdin')) sys.exit(1) start = time.clock() collection_rules, rm_conf = pc.get_conf(InsightsClient.options.update, stdin_config) collection_elapsed = (time.clock() - start) logger.debug("Rules configuration loaded. Elapsed time: %s", collection_elapsed) individual_archives = [] for t in targets: # defaults archive = None container_connection = None mp = None obfuscate = None # archive metadata archive_meta = {} try: if t['type'] == 'docker_image': container_connection = open_image(t['name']) logging_name = 'Docker image ' + t['name'] archive_meta['docker_id'] = t['name'] archive_meta['display_name'] = docker_display_name( t['name'], t['type'].replace('docker_', '')) logger.debug('Docker display_name: %s', archive_meta['display_name']) logger.debug('Docker docker_id: %s', archive_meta['docker_id']) if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) sys.exit(1) elif t['type'] == 'docker_container': container_connection = open_container(t['name']) logging_name = 'Docker container ' + t['name'] archive_meta['docker_id'] = t['name'] archive_meta['display_name'] = docker_display_name( t['name'], t['type'].replace('docker_', '')) logger.debug('Docker display_name: %s', archive_meta['display_name']) logger.debug('Docker docker_id: %s', archive_meta['docker_id']) if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) sys.exit(1) elif t['type'] == 'host': logging_name = determine_hostname() archive_meta['display_name'] = determine_hostname( InsightsClient.options.display_name) else: logger.error('Unexpected analysis target: %s', t['type']) sys.exit(1) archive_meta['type'] = t['type'].replace('docker_', '') archive_meta['product'] = 'Docker' archive_meta['system_id'] = generate_analysis_target_id( t['type'], t['name']) collection_start = time.clock() archive = InsightsArchive( compressor=InsightsClient.options.compressor if not InsightsClient.options.container_mode else "none", target_name=t['name']) atexit.register(_delete_archive, archive) dc = DataCollector(archive, InsightsClient.config, mountpoint=mp, target_name=t['name'], target_type=t['type']) logger.info('Starting to collect Insights data for %s', logging_name) dc.run_collection(collection_rules, rm_conf, branch_info) elapsed = (time.clock() - start) logger.debug("Data collection complete. Elapsed time: %s", elapsed) obfuscate = InsightsClient.config.getboolean(APP_NAME, "obfuscate") # include rule refresh time in the duration collection_duration = (time.clock() - collection_start) + collection_elapsed # add custom metadata about a host if provided by from_file # use in the OSE case if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) if 'metadata' in stdin_config: archive.add_metadata_to_archive( json.dumps(stdin_config['metadata']), 'metadata.json') if InsightsClient.options.no_tar_file: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc tar_file = dc.done(collection_rules, rm_conf) # add archives to list of individual uploads archive_meta['tar_file'] = tar_file individual_archives.append(archive_meta) finally: # called on loop iter end or unexpected exit if container_connection: container_connection.close() # if multiple targets (container mode), add all archives to single archive # if InsightsClient.options.container_mode: if False: # we only run single collections now (not the uber archives), bypass this full_archive = InsightsArchive( compressor=InsightsClient.options.compressor) for a in individual_archives: shutil.copy(a['tar_file'], full_archive.archive_dir) # don't want insights_commands in meta archive shutil.rmtree(full_archive.cmd_dir) metadata = _create_metadata_json(individual_archives) full_archive.add_metadata_to_archive(json.dumps(metadata), 'metadata.json') full_tar_file = full_archive.create_tar_file(full_archive=True) # if only one target (regular mode), just upload one else: full_archive = archive full_tar_file = tar_file if InsightsClient.options.offline or InsightsClient.options.no_upload: handle_file_output(full_tar_file, full_archive) return rc # do the upload rc = _do_upload(pconn, full_tar_file, logging_name, collection_duration) if InsightsClient.options.keep_archive: logger.info('Insights data retained in %s', full_tar_file) return rc if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(full_tar_file)) full_archive.delete_archive_dir() return rc