def testRunWithError(self): collector = DataCollector(in_path=IN_FILE_BAD, ot_path_data=OT_FILE_DATA, ot_path_doc=OT_FILE_DOC) collector.run() df = pd.read_csv(OT_FILE_DATA) self.assertEqual(len(df["Biomodel_Id"]), 3)
def run_random_search(verbose, num_diff_experiments, num_repeat_experiment, allow_duplicates=False, df_path=None, overwrite=True, data_to_collect=POSSIBLE_DATA, MVP_key='waitingTime', save_model=True): grid = load_constants('constants/constants-grid.json') if not allow_duplicates: _, num_choices = get_num_grid_choices(grid) num_diff_experiments = min(num_choices, num_diff_experiments) # Make grid choice generator grid_choice_gen = grid_choices_random(grid, num_diff_experiments) for diff_experiment, constants in enumerate(grid_choice_gen): data_collector_obj = DataCollector( data_to_collect, MVP_key, constants, 'test' if constants['agent']['agent_type'] == 'rule' else 'eval', df_path, overwrite if diff_experiment == 0 else False, verbose) for same_experiment in range(num_repeat_experiment): print(' --- Running experiment {}.{} / {}.{} --- '.format( diff_experiment + 1, same_experiment + 1, num_diff_experiments, num_repeat_experiment)) if save_model: data_collector_obj.set_save_model_path( 'models/saved_models/random_{}-{}.pt'.format( diff_experiment + 1, same_experiment + 1)) run_experiment(diff_experiment + 1, same_experiment + 1, constants, data_collector_obj)
def setup_data_threads(self, sample_time = None, decimation = None, start_threads = False): if self.is_slave: if not self.datastream_queue: self.datastream_queue = [] # TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller. if not self.datastream_poller: self.datastream_poller = DataPoller(self.datastream_queue, decimation) # TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati if not self.datastream_collector: # TODO: Fermare i thread alla chiusura del target self.datastream_collector = DataCollector(self.config.getAttr('address'), self.config.getAttr('slave_data_port'), self.datastream_queue) if start_threads: print("Starting data threads...") if sample_time: self.datastream_poller.updateSampleTime(sample_time) self.datastream_poller.start() self.datastream_collector.start() print("Data threads started...") else: print("Can't start data threads without a sample time") return False return True else: print("Can't setup the data stream threads on a not-slave server") return False
def start(self): with self.__lock: self.__need_to_stop_flag = False self.__run_browser() once_authorized = False self.__load_url(YandexWordstatParser.BASE_URL) for phrase in self.__phrases: self.__type_in_phrase_to_search_field_and_press_find(phrase) if not once_authorized: try: self.__authorize_on_redirected_form() except TimeoutException: self.__authorize() once_authorized = True # wait until DOM is not built to avoid selenium.common.exceptions.StaleElementReferenceException time.sleep(1) DataCollector.store_wordstat_data(self.__extract_keywords(), phrase) YandexWordstatParser.__random_wait(3000, 15000) self.__driver.quit() self.on_completed.emit()
def main(): args = get_args() data_collector_obj = DataCollector(args.debug) if args.debug: print("Got arguments " + str(args.years) + ", " + str(args.players) + ", " + args.mode + ", " + str(args.singles)) data_collector_obj.scrape_last_n_years_of_k_players( args.years, args.players, args.mode, "singles" if args.singles else "doubles")
def setUp(self): data_collector = DataCollector(config=config) self.data_after = data_collector.data_collector() path = 'devlab/tests' file_name = config.rollback_params['data_file_names']['PRE'] pre_file_path = os.path.join(data_collector.main_folder, path, file_name) with open(pre_file_path, "r") as f: self.pre_data = yaml.load(f)
def start(self): with self.__lock: self.__need_to_stop_flag = False self.__run_browser() for phrase in self.__phrases: result = [] for city_code in self.__city_codes: if self.__need_to_stop(): break variables = { YandexParser.TEXT_VARIABLE_NAME: phrase, YandexParser.CITY_CODE_VARIABLE_NAME: city_code } self.__load_url(YandexParser.BASE_URL + '?' + urllib.parse.urlencode(variables)) def parse_ads_city_phrase_loop(): nonlocal result while len(result) < self.__ads_count_for_one_key: result += self.__parse_loaded_page( self.__ads_count_for_one_key - len(result)) if len(result) >= self.__ads_count_for_one_key: break else: try: next_page_link = self.__driver.find_element_by_class_name( 'pager__item_kind_next') time.sleep(self.__pause_time_between_requests / 1000) self.__driver.execute_script('console.clear()') next_page_link.click() except NoSuchElementException as error: self.__logger.warning( 'There is no more ads for phrase \'%s\'' % phrase) self.__logger.warning(error) break parse_ads_city_phrase_loop() self.__logger.info(result) result = self._remove_duplicated_hosts(result) DataCollector.store_ad_search_data('YandexParser', phrase, result) self.__driver.quit() self.on_completed.emit()
def __init__(self, api_host, gatsby_host, alias_host, num_threads): self.select_clause = "mpidStr AS \'mpid\', priceRange, aggregatedRatings, modelTitle AS \'title\', brandName, categoryNamePath, searchScore, brandName, storeId, image" self.page_size = 500 self.country_code = 356 self.gatsby_query = ProductGatsbyQuery(self.select_clause, self.page_size, self.country_code, "/products/search2", gatsby_host) self.gatsbyPB_query = ProductGatsbyQuery(self.select_clause, self.page_size, self.country_code, "/products/search2", gatsby_host) self.api_query = ProductApiQuery("http", api_host, "/v2.1/search", "IN", 50) self.alias_service = DataCollector( ProductAliasQuery("http", alias_host, "/search", "IN")) self.ranking_model = RankingModel() self.gatsby_queries = Queue(2) self.gatsbyPB_queries = Queue(2) self.api_queries = Queue(2) worker = Worker(DataCollector(self.gatsby_query), self.ranking_model, self.gatsby_queries) # worker.daemon = True print "currently running thread:\t", worker.getName() worker.start() worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model, self.gatsbyPB_queries) print "currently running thread:\t", worker.getName() # worker.daemon = True worker.start() # for thread in range(2): # worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model, self.gatsbyPB_queries) # print "currently running thread:\t", worker.getName() # worker.daemon = True # worker.start() # # for thread in range(2): # worker = Worker(DataCollector(self.gatsby_query), self.ranking_model, self.gatsby_queries) # worker.daemon = True # worker.start() worker = Worker(DataCollector(self.api_query), self.ranking_model, self.api_queries) print "currently running thread:\t", worker.getName() # worker.daemon = True worker.start()
def setup_data_threads(self, sample_time=None, decimation=None, start_threads=False): if self.is_slave: if not self.datastream_queue: self.datastream_queue = [] # TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller. if not self.datastream_poller: self.datastream_poller = DataPoller(self.datastream_queue, decimation) # TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati if not self.datastream_collector: # TODO: Fermare i thread alla chiusura del target self.datastream_collector = DataCollector( self.config.getAttr("address"), self.config.getAttr("slave_data_port"), self.datastream_queue ) if start_threads: print ("Starting data threads...") if sample_time: self.datastream_poller.updateSampleTime(sample_time) self.datastream_poller.start() self.datastream_collector.start() print ("Data threads started...") else: print ("Can't start data threads without a sample time") return False return True else: print ("Can't setup the data stream threads on a not-slave server") return False
def register(config, group_id=None): """ Do registration using basic auth """ username = config.get(APP_NAME, 'username') password = config.get(APP_NAME, 'password') if (((username == "") and (password == "") and (config.get(APP_NAME, 'authmethod') == 'BASIC')) and not (config.get(APP_NAME, 'auto_config'))): # Get input from user print "Please enter your Red Hat Customer Portal Credentials" sys.stdout.write('User Name: ') username = raw_input().strip() password = getpass.getpass() sys.stdout.write("Would you like to save these credentials? (y/n) ") save = raw_input().strip() config.set(APP_NAME, 'username', username) config.set(APP_NAME, 'password', password) logger.debug("savestr: %s", save) if save.lower() == "y" or save.lower() == "yes": logger.debug("writing user/pass to config file") cmd = ("/bin/sed -e 's/^username.*=.*$/username="******"/' " + "-e 's/^password.*=.*$/password="******"/' " + constants.default_conf_file) status = DataCollector().run_command_get_output(cmd, nolog=True) config_file = open(constants.default_conf_file, 'w') config_file.write(status['output']) config_file.flush() pconn = InsightsConnection(config) return pconn.register(group_id)
def __init__(self, data_collector=None): if data_collector is None: data_collector = DataCollector() else: self.data_collector = data_collector self.bigrams = dict() self.get_unique_bigrams()
def mean_std_of_states(): env = MountainCarWithResetEnv() samples_to_collect = 100000 states, actions, rewards, next_states, done_flags = DataCollector(env).collect_data(samples_to_collect) all_states = np.concatenate((states, next_states)) states_mean = np.mean(all_states, axis=0) states_std = np.std(all_states, axis=0) print("states_mean: {}, states_std: {}".format(states_mean, states_std))
class Preprocessing: def __init__(self): self.di = DataCollector() def to_upper(self): data = self.di.collect_data() upper = data['data'].upper() return upper
def __init__(self): with DataCollector() as collector: result_channels = collector.get_telemetry_outputs() with DataServer(result_channels) as transmitter: print("Press ctrl+c to exit") signal.signal(signal.SIGINT, self.exit_handler) # TODO signal.pause() is not available on Windows systems signal.pause()
def perfrom_experiment(self, experiment_name='test', movement_list=[]): # 1. We save the background image: dc = DataCollector(only_one_shot=False, automatic=True) dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=False, save=True, directory=experiment_name+'/air', iteration=-1) print "Air data gathered" # 2. We perfomr the experiment: i = 0 if not os.path.exists(experiment_name): # If the directory does not exist, we create it os.makedirs(experiment_name) for movement in movement_list: path = experiment_name + '/p_' + str(i) + '/' self.palpate(speed=40, force_list=self.force_list, save=True, path=path) self.move_cart_mm(movement[0], movement[1], movement[2]) time.sleep(6) i += 1 path = experiment_name + '/p_' + str(i) + '/' self.palpate(speed=40, force_list=self.force_list, save=True, path=path)
def training_the_model(samples_to_collect=100000, seed=100): number_of_kernels_per_dim = [10, 8] gamma = 0.999 w_updates = 20 evaluation_number_of_games = 50 evaluation_max_steps_per_game = 300 np.random.seed(seed) env = MountainCarWithResetEnv() # collect data states, actions, rewards, next_states, done_flags = DataCollector( env).collect_data(samples_to_collect) # get data success rate data_success_rate = np.sum(rewards) / len(rewards) print(f'Data Success Rate {data_success_rate}') # standardize data data_transformer = DataTransformer() data_transformer.set_using_states( np.concatenate((states, next_states), axis=0)) states = data_transformer.transform_states(states) next_states = data_transformer.transform_states(next_states) # process with radial basis functions feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim) # encode all states: encoded_states = feature_extractor.encode_states_with_radial_basis_functions( states) encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions( next_states) # set a new linear policy linear_policy = LinearPolicy(feature_extractor.get_number_of_features(), 3, True) # but set the weights as random linear_policy.set_w(np.random.uniform(size=linear_policy.w.shape)) # start an object that evaluates the success rate over time evaluator = GamePlayer(env, data_transformer, feature_extractor, linear_policy) success_rate_vs_iteration = list() for lspi_iteration in range(w_updates): print(f'Starting LSPI iteration {lspi_iteration}') new_w = compute_lspi_iteration(encoded_states, encoded_next_states, actions, rewards, done_flags, linear_policy, gamma) norm_diff = linear_policy.set_w(new_w) success_rate = evaluator.play_games(evaluation_number_of_games, evaluation_max_steps_per_game) success_rate_vs_iteration.append(success_rate) if norm_diff < 0.00001: break print('LSPI Done') return success_rate_vs_iteration
def palpate(self, speed=40, force_list=[1, 10, 20, 40], save=False, path=''): # 0. We create the directory if save is True and not os.path.exists(path): # If the directory does not exist, we create it os.makedirs(path) # 1. We get and save the cartesian coord. dc = DataCollector(only_one_shot=False, automatic=True) cart = dc.getCart() if save is True: np.save(path + '/cart.npy', cart) # 2. We get wsg forces and gs images at every set force and store them i = 0 for force in force_list: self.close_gripper_f(grasp_speed=speed, grasp_force=force) print "Applying: " + str(force) time.sleep(1.0) dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=True, save=save, directory=path, iteration=i) self.open_gripper() time.sleep(1.0) i += 1
def __init__(self): self.screen = MeteoScreen() self.collector = DataCollector() self.data = InMemorySensorDataLogger() self.collector.add_logger(self.data) file_logger = ToFileSensorDataLogger('./db') self.collector.add_logger(file_logger) now = datetime.datetime.now() last24h_data = file_logger.load_data(now - datetime.timedelta(days=1), now) self.data.set_data(last24h_data) self.screen_update_interval = 10 self.current_screen_draw = self.draw_screen1 self.update_timer_stop = threading.Event() self.screen_update_lock = threading.RLock() def timer(): while not self.update_timer_stop.is_set(): time.sleep(self.screen_update_interval) self.update_screen() self.timer_thread = threading.Thread(target=timer)
class TestDataCollector(unittest.TestCase): def setUp(self): self.collector = DataCollector(in_path=IN_FILE, ot_path_data=OT_FILE_DATA, ot_path_doc=OT_FILE_DOC) def testConstructor(self): if IGNORE_TEST: return self.assertEqual(self.collector._ot_path_data, OT_FILE_DATA) def testRun(self): self.collector.run() df = pd.read_csv(OT_FILE_DATA) self.assertEqual(len(df["Biomodel_Id"]), 2) def testRunWithError(self): collector = DataCollector(in_path=IN_FILE_BAD, ot_path_data=OT_FILE_DATA, ot_path_doc=OT_FILE_DOC) collector.run() df = pd.read_csv(OT_FILE_DATA) self.assertEqual(len(df["Biomodel_Id"]), 3)
def run_normal(verbose, num_experiments=1, df_path=None, overwrite=True, data_to_collect=POSSIBLE_DATA, MVP_key='waitingTime', save_model=True, load_model_file=None): # if loading, then dont save if load_model_file: save_model = False if not df_path: df_path = 'run-data.xlsx' # def. path # Load constants constants = load_constants('constants/constants.json') data_collector_obj = DataCollector( data_to_collect, MVP_key, constants, 'test' if constants['agent']['agent_type'] == 'rule' or load_model_file else 'eval', df_path, overwrite, verbose) loaded_model = None if load_model_file: loaded_model = torch.load('models/saved_models/' + load_model_file) for exp in range(num_experiments): print(' --- Running experiment {} / {} --- '.format( exp + 1, num_experiments)) if save_model: data_collector_obj.set_save_model_path( 'models/saved_models/normal_{}.pt'.format(exp + 1)) run_experiment(exp + 1, None, constants, data_collector_obj, loaded_model=loaded_model)
def collect_data_and_upload(config, options): """ All the heavy lifting done here """ pconn = InsightsConnection(config) pconn.check_registration() branch_info = pconn.branch_info() pc = InsightsConfig(config, pconn) dc = DataCollector() logger.info('Collecting Insights data') start = time.clock() dynamic_config = pc.get_conf(options.update) elapsed = (time.clock() - start) logger.debug("Dynamic Config Elapsed Time: %s", elapsed) start = time.clock() dc.run_commands(dynamic_config) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(dynamic_config) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") if not options.no_tar_file: tar_file = dc.done(config) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') pconn.upload_archive(tar_file) logger.info( 'Check https://access.redhat.com/labs/insights in an hour') if not obfuscate and not options.keep_archive: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: logger.info('See Insights data in %s', tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir)
if __name__ == '__main__': samples_to_collect = 100000 # samples_to_collect = 150000 # samples_to_collect = 10000 number_of_kernels_per_dim = [10, 8] gamma = 0.99 w_updates = 100 evaluation_number_of_games = 10 evaluation_max_steps_per_game = 1000 np.random.seed(123) # np.random.seed(234) env = MountainCarWithResetEnv() # collect data states, actions, rewards, next_states, done_flags = DataCollector( env).collect_data(samples_to_collect) # get data success rate data_success_rate = np.sum(rewards) / len(rewards) print(f'success rate {data_success_rate}') # standardize data data_transformer = DataTransformer() data_transformer.set_using_states( np.concatenate((states, next_states), axis=0)) states = data_transformer.transform_states(states) next_states = data_transformer.transform_states(next_states) # process with radial basis functions feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim) # encode all states: encoded_states = feature_extractor.encode_states_with_radial_basis_functions( states) encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions(
#!/usr/bin/python from data_collector import DataCollector import rospy import sys if __name__ == "__main__": rospy.init_node("data_collector") collector = DataCollector() rospy.spin() collector.dump()
def test_makes_get_request_to_node_fullpath(self, request): some_path = "http://localhost:1231" node = Mock(**{"full_path.return_value": some_path}) data = DataCollector.get_node_info(node) request.assert_called_with(some_path)
def test_gets_data_from_node(self, node_data): data = DataCollector.get_node_info(Mock()) assert_that(data, is_(SAMPLE_NODE_DATA))
def collect_data_and_upload(config, options, rc=0): """ All the heavy lifting done here """ collection_start = time.clock() pconn = InsightsConnection(config) try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information", options) except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information", options) pc = InsightsConfig(config, pconn) archive = InsightsArchive(compressor=options.compressor) dc = DataCollector(archive) # register the exit handler here to delete the archive atexit.register(handle_exit, archive, options.keep_archive or options.no_upload) stdin_config = json.load(sys.stdin) if options.from_stdin else {} start = time.clock() collection_rules, rm_conf = pc.get_conf(options.update, stdin_config) elapsed = (time.clock() - start) logger.debug("Collection Rules Elapsed Time: %s", elapsed) start = time.clock() logger.info('Starting to collect Insights data') dc.run_commands(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") collection_duration = (time.clock() - collection_start) if not options.no_tar_file: tar_file = dc.done(config, rm_conf) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') for tries in range(options.retries): upload = pconn.upload_archive(tar_file, collection_duration) if upload.status_code == 201: write_lastupload_file() logger.info("Upload completed successfully!") break elif upload.status_code == 412: pconn.handle_fail_rcs(upload) else: logger.error("Upload attempt %d of %d failed! Status Code: %s", tries + 1, options.retries, upload.status_code) if tries + 1 != options.retries: logger.info("Waiting %d seconds then retrying", constants.sleep_time) time.sleep(constants.sleep_time) else: logger.error("All attempts to upload have failed!") logger.error("Please see %s for additional information", constants.default_log_file) rc = 1 if (not obfuscate and not options.keep_archive) or options.no_upload: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: handle_file_output(options, tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc
#maybe? ''' def get_possible_seed_words(self, length): all_tokens = get_unique_tokens_count(self.data_collector.corpus) possible_seeds = [] for tkn in all_tokens: if (len(tkn[0]) == int(length)): possible_seeds.append(tkn[0]) return possible_seeds''' #collect data/clean it collector = DataCollector() collector.clean_corpus() #print(collector.corpus) bigram_model = BigramModel(collector) dir_name = 'txt_examples' for filename in os.listdir(dir_name): print('INFO: generating text for {fname}'.format(fname=filename)) word_lengths = [] f = open('{directory}/{file}'.format(directory=dir_name, file=filename), 'r') for line in f:
pos = tuple(frame.position[:3]) rot = tfx.tb_angles(frame.rotation) rot = (rot.yaw_deg, rot.pitch_deg, rot.roll_deg) pickle.dump({'pos': pos, 'rot': rot, 'estimate': estimate}, f) f.close() psm1 = robot("PSM1") psm1.open_gripper(80) time.sleep(2) d = DataCollector() time.sleep(2) img = cv2.medianBlur(d.left_image[:, 850:], 13) img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.bilateralFilter(img, 11, 17, 17) output = cv2.Canny(img, 100, 200) (cnts, _) = cv2.findContours(output.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cnts = sorted(cnts, key=cv2.contourArea, reverse=True) for c in cnts: # approximate the contour peri = cv2.arcLength(c, True)
def collect_data_and_upload(config, options): """ All the heavy lifting done here """ pconn = InsightsConnection(config) try: branch_info = pconn.branch_info() except requests.ConnectionError: logger.error("ERROR: Could not connect to determine branch information") sys.exit() except LookupError: logger.error("ERROR: Could not determine branch information") sys.exit() pc = InsightsConfig(config, pconn) dc = DataCollector() start = time.clock() collection_rules, rm_conf = pc.get_conf(options.update) elapsed = (time.clock() - start) logger.debug("Collection Rules Elapsed Time: %s", elapsed) start = time.clock() logger.info('Starting to collect Insights data') dc.run_commands(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") if not options.no_tar_file: tar_file = dc.done(config, rm_conf) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') for tries in range(options.retries): status = pconn.upload_archive(tar_file) if status == 201: logger.info("Upload completed successfully!") break else: logger.error("Upload attempt %d of %d failed! Status Code: %s", tries+1, options.retries, status) if tries +1 != options.retries: logger.info("Waiting %d seconds then retrying", constants.sleep_time) time.sleep(constants.sleep_time) else: logger.error("All attempts to upload have failed!") logger.error("Please see %s for additional information", constants.default_log_file) if not obfuscate and not options.keep_archive: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: logger.info('See Insights data in %s', tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir)
def collect(self, dir): DataCollector.collect(self, dir) try: self.total_authors = int(getpipeoutput(["git log", "git shortlog -s", "wc -l"])) except: self.total_authors = 0 # self.total_lines = int(getoutput('git-ls-files -z |xargs -0 cat |wc -l')) self.activity_by_hour_of_day = {} # hour -> commits self.activity_by_day_of_week = {} # day -> commits self.activity_by_month_of_year = {} # month [1-12] -> commits self.activity_by_hour_of_week = {} # weekday -> hour -> commits self.activity_by_hour_of_day_busiest = 0 self.activity_by_hour_of_week_busiest = 0 self.activity_by_year_week = {} # yy_wNN -> commits self.activity_by_year_week_peak = 0 self.authors = ( {} ) # name -> {commits, first_commit_stamp, last_commit_stamp, last_active_day, active_days, lines_added, lines_removed} # domains self.domains = {} # domain -> commits # author of the month self.author_of_month = {} # month -> author -> commits self.author_of_year = {} # year -> author -> commits self.commits_by_month = {} # month -> commits self.commits_by_year = {} # year -> commits self.first_commit_stamp = 0 self.last_commit_stamp = 0 self.last_active_day = None self.active_days = set() # lines self.total_lines = 0 self.total_lines_added = 0 self.total_lines_removed = 0 # timezone self.commits_by_timezone = {} # timezone -> commits # tags self.tags = {} lines = getpipeoutput(["git show-ref --tags"]).split("\n") for line in lines: if len(line) == 0: continue (hash, tag) = line.split(" ") tag = tag.replace("refs/tags/", "") output = getpipeoutput(['git log "%s" --pretty=format:"%%at %%an" -n 1' % hash]) if len(output) > 0: parts = output.split(" ") stamp = 0 try: stamp = int(parts[0]) except ValueError: stamp = 0 self.tags[tag] = { "stamp": stamp, "hash": hash, "date": datetime.datetime.fromtimestamp(stamp).strftime("%Y-%m-%d"), "commits": 0, "authors": {}, } # collect info on tags, starting from latest tags_sorted_by_date_desc = map( lambda el: el[1], reversed(sorted(map(lambda el: (el[1]["date"], el[0]), self.tags.items()))) ) prev = None for tag in reversed(tags_sorted_by_date_desc): cmd = 'git shortlog -s "%s"' % tag if prev != None: cmd += ' "^%s"' % prev output = getpipeoutput([cmd]) if len(output) == 0: continue prev = tag for line in output.split("\n"): parts = re.split("\s+", line, 2) commits = int(parts[1]) author = parts[2] self.tags[tag]["commits"] += commits self.tags[tag]["authors"][author] = commits # Collect revision statistics # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'" lines = getpipeoutput(['git rev-list --pretty=format:"%at %ai %an <%aE>" HEAD', "grep -v ^commit"]).split("\n") for line in lines: parts = line.split(" ", 4) author = "" try: stamp = int(parts[0]) except ValueError: stamp = 0 timezone = parts[3] author, mail = parts[4].split("<", 1) author = author.rstrip() mail = mail.rstrip(">") domain = "?" if mail.find("@") != -1: domain = mail.rsplit("@", 1)[1] date = datetime.datetime.fromtimestamp(float(stamp)) # First and last commit stamp if self.last_commit_stamp == 0: self.last_commit_stamp = stamp self.first_commit_stamp = stamp # activity # hour hour = date.hour self.activity_by_hour_of_day[hour] = self.activity_by_hour_of_day.get(hour, 0) + 1 # most active hour? if self.activity_by_hour_of_day[hour] > self.activity_by_hour_of_day_busiest: self.activity_by_hour_of_day_busiest = self.activity_by_hour_of_day[hour] # day of week day = date.weekday() self.activity_by_day_of_week[day] = self.activity_by_day_of_week.get(day, 0) + 1 # domain stats if domain not in self.domains: self.domains[domain] = {} # commits self.domains[domain]["commits"] = self.domains[domain].get("commits", 0) + 1 # hour of week if day not in self.activity_by_hour_of_week: self.activity_by_hour_of_week[day] = {} self.activity_by_hour_of_week[day][hour] = self.activity_by_hour_of_week[day].get(hour, 0) + 1 # most active hour? if self.activity_by_hour_of_week[day][hour] > self.activity_by_hour_of_week_busiest: self.activity_by_hour_of_week_busiest = self.activity_by_hour_of_week[day][hour] # month of year month = date.month self.activity_by_month_of_year[month] = self.activity_by_month_of_year.get(month, 0) + 1 # yearly/weekly activity yyw = date.strftime("%Y-%W") self.activity_by_year_week[yyw] = self.activity_by_year_week.get(yyw, 0) + 1 if self.activity_by_year_week_peak < self.activity_by_year_week[yyw]: self.activity_by_year_week_peak = self.activity_by_year_week[yyw] # author stats if author not in self.authors: self.authors[author] = {} # commits if "last_commit_stamp" not in self.authors[author]: self.authors[author]["last_commit_stamp"] = stamp self.authors[author]["first_commit_stamp"] = stamp self.authors[author]["commits"] = self.authors[author].get("commits", 0) + 1 # author of the month/year yymm = date.strftime("%Y-%m") if yymm in self.author_of_month: self.author_of_month[yymm][author] = self.author_of_month[yymm].get(author, 0) + 1 else: self.author_of_month[yymm] = {} self.author_of_month[yymm][author] = 1 self.commits_by_month[yymm] = self.commits_by_month.get(yymm, 0) + 1 yy = date.year if yy in self.author_of_year: self.author_of_year[yy][author] = self.author_of_year[yy].get(author, 0) + 1 else: self.author_of_year[yy] = {} self.author_of_year[yy][author] = 1 self.commits_by_year[yy] = self.commits_by_year.get(yy, 0) + 1 # authors: active days yymmdd = date.strftime("%Y-%m-%d") if "last_active_day" not in self.authors[author]: self.authors[author]["last_active_day"] = yymmdd self.authors[author]["active_days"] = 1 elif yymmdd != self.authors[author]["last_active_day"]: self.authors[author]["last_active_day"] = yymmdd self.authors[author]["active_days"] += 1 # project: active days if yymmdd != self.last_active_day: self.last_active_day = yymmdd self.active_days.add(yymmdd) # timezone self.commits_by_timezone[timezone] = self.commits_by_timezone.get(timezone, 0) + 1 # TODO Optimize this, it's the worst bottleneck # outputs "<stamp> <files>" for each revision self.files_by_stamp = {} # stamp -> files revlines = getpipeoutput(['git rev-list --pretty=format:"%at %T" HEAD', "grep -v ^commit"]).strip().split("\n") lines = [] for revline in revlines: time, rev = revline.split(" ") linecount = self.getFilesInCommit(rev) lines.append("%d %d" % (int(time), linecount)) self.total_commits = len(lines) for line in lines: parts = line.split(" ") if len(parts) != 2: continue (stamp, files) = parts[0:2] try: self.files_by_stamp[int(stamp)] = int(files) except ValueError: print 'Warning: failed to parse line "%s"' % line # extensions self.extensions = {} # extension -> files, lines lines = getpipeoutput(["git ls-tree -r -z HEAD"]).split("\000") self.total_files = len(lines) for line in lines: if len(line) == 0: continue parts = re.split("\s+", line, 4) sha1 = parts[2] filename = parts[3] if filename.find(".") == -1 or filename.rfind(".") == 0: ext = "" else: ext = filename[(filename.rfind(".") + 1) :] if len(ext) > self.conf["max_ext_length"]: ext = "" if ext not in self.extensions: self.extensions[ext] = {"files": 0, "lines": 0} self.extensions[ext]["files"] += 1 try: self.extensions[ext]["lines"] += self.getLinesInBlob(sha1) except: print 'Warning: Could not count lines for file "%s"' % line # line statistics # outputs: # N files changed, N insertions (+), N deletions(-) # <stamp> <author> self.changes_by_date = {} # stamp -> { files, ins, del } lines = getpipeoutput(['git log --shortstat --pretty=format:"%at %an"']).split("\n") lines.reverse() files = 0 inserted = 0 deleted = 0 total_lines = 0 author = None for line in lines: if len(line) == 0: continue # <stamp> <author> if line.find("files changed,") == -1: pos = line.find(" ") if pos != -1: try: (stamp, author) = (int(line[:pos]), line[pos + 1 :]) self.changes_by_date[stamp] = { "files": files, "ins": inserted, "del": deleted, "lines": total_lines, } if author not in self.authors: self.authors[author] = {"lines_added": 0, "lines_removed": 0} self.authors[author]["lines_added"] = self.authors[author].get("lines_added", 0) + inserted self.authors[author]["lines_removed"] = self.authors[author].get("lines_removed", 0) + deleted except ValueError: print 'Warning: unexpected line "%s"' % line else: print 'Warning: unexpected line "%s"' % line else: numbers = re.findall("\d+", line) if len(numbers) == 3: (files, inserted, deleted) = map(lambda el: int(el), numbers) total_lines += inserted total_lines -= deleted self.total_lines_added += inserted self.total_lines_removed += deleted else: print 'Warning: failed to handle line "%s"' % line (files, inserted, deleted) = (0, 0, 0) # self.changes_by_date[stamp] = { 'files': files, 'ins': inserted, 'del': deleted } self.total_lines = total_lines
new_SD_list, new_not_SD = loop_through_ids( self.ids_social_distance) assert len(new_SD_list) == 0 # Switch people for id in new_SD: self.ids_not_social_distance.remove(id) self.ids_social_distance.add(id) for id in new_not_SD: self.ids_social_distance.remove(id) self.ids_not_social_distance.add(id) if render: pygame.display.flip() screen.fill((0, 0, 0)) # Frames per second if self.render_C['fps']: clock.tick(self.render_C['fps']) self.data_collect.reset(t + 1, last=True) if __name__ == '__main__': constants = json.load(open('constants.json')) # Can save a run as an experiment which saves the data, visualizations and constants in a experiments directory data_collect = DataCollector(constants, save_experiment=True, print_visualizations=True) # Can print data (look at `data_options` at top of `data_collector.py` for options) and how often to print data_collect.set_print_options(basic_to_print=['S', 'I', 'R', 'death'], frequency=1) CA = CellularAutomation(constants, data_collect) # Can render each timestep with pygame CA.run(render=True)
class Target: ## @var config # A configurator::Configurator object ## @var connection # A connector::Connector instance used to send commands to XML-RPC connection ## @var is_slave # Shows if this is a slave target ## @var parameters # The target parameters ## @var datastream_queue # The queue where DataCollector thread puts data and observers get it ## @var datastream_poller # The DataPoller thread ## @var datastream_collector # The DataCollector thread ## @var signal_structure # The structure of the signal obtained with getSignalStructure ## @var is_connected # Checks if the Target is connected or not. Actually the status is simply set bu connect() and disconnect() ## Target initializer. def __init__(self, slave=False, protocol=None, address=None, port=None, target_name=None): self.config = Configurator() self.is_connected = False self.is_slave = slave self.parameters = None self.datastream_queue = None self.datastream_poller = None self.datastream_collector = None self.signal_structure = None # Sets basic configurations if protocol: self.config.setAttr("protocol", protocol) if address: self.config.setAttr("address", address) if port: self.config.setAttr("port", port) if target_name: self.config.setAttr("target_name", target_name) # Initializes connection self.connection = Connector(self.config) ## Prints a summary of the target and its status # TODO: Decidere che informazioni restituire def getInfo(self): print "Target name: %s" % self.config.getAttr("target_name") print "Connection status: %s" % ("Connected" if self.is_connected else "Disconnected") print "Connection request: %s://%s:%s" % ( self.config.getAttr("protocol"), self.config.getAttr("address"), self.config.getAttr("port"), ) ## Setups the queue and the threads needed to read the data stream # @param sample_time The sample time of the signal # @param decimation The decimation to use # @start_threads The flag given by startData() to let the data threads start (False if this method is called by addObserver()) # @return The boolean result of the operation def setup_data_threads(self, sample_time=None, decimation=None, start_threads=False): if self.is_slave: if not self.datastream_queue: self.datastream_queue = [] # TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller. if not self.datastream_poller: self.datastream_poller = DataPoller(self.datastream_queue, decimation) # TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati if not self.datastream_collector: # TODO: Fermare i thread alla chiusura del target self.datastream_collector = DataCollector( self.config.getAttr("address"), self.config.getAttr("slave_data_port"), self.datastream_queue ) if start_threads: print ("Starting data threads...") if sample_time: self.datastream_poller.updateSampleTime(sample_time) self.datastream_poller.start() self.datastream_collector.start() print ("Data threads started...") else: print ("Can't start data threads without a sample time") return False return True else: print ("Can't setup the data stream threads on a not-slave server") return False ## Stops the data threads # @return The boolean result of the operation def stop_data_threads(self): if self.is_slave: if self.datastream_collector: print ("Shutting down DataCollector thread") self.datastream_collector.stop() self.datastream_collector.join(15.0) if self.datastream_collector.is_alive(): print ("Can't stop the DataCollector thread, timeout occurs") else: print ("DataCollector thread was shut down") if self.datastream_poller: print ("Shutting down DataPoller thread") self.datastream_poller.stop() self.datastream_poller.join(15.0) if self.datastream_poller.is_alive(): print ("Can't stop the DataPoller thread, timeout occurs") else: print ("DataPoller thread was shut down") # Clears vars self.datastream_poller = None self.datastream_collector = None self.datastream_queue = None return True else: print ("Can't stop the data stream threads on a not-slave server") return False ## Add a new observer "update" method to the list of observers. # @param update_method Is the method of the observer which will be called when new data arrives. The method name is arbitrary, but it must accept a string parameter # @param signal_number The optional channel to register the observer to # @param decimation The decimation to use # @return The ID of the observer, to be used to remove the observer def addObserver(self, update_method, decimation=None, signal_number=None): self.setup_data_threads(None, decimation) return self.datastream_poller.addObserver(update_method, signal_number) ## Removes the observer # @param observer_id The ID of the observer to remove # @return False if the observer isn't in the observers list, True if it was correctly removed def removeObserver(self, observer_id): return self.datastream_poller.removeObserver(observer_id) ## Connect to the target server. Checks if master or server and call the right method on connector::Connector # @param password The (optional) password to catch a running master connection # @return The boolean result of the operation def connect(self, password=None): if self.is_slave: if ( self.connection.connectSlave() ): # TODO: Cambiare il metodo di connessione non passando tutto il Configurator ma usando i dati passati nel costruttore di Connector self.is_connected = True self.signal_structure = self.getSignalStructure() return True else: return False else: if password: result = self.connection.Connection_Request( password ) # Passing self.config to let the Connector set some connection data (e.g. the session_id) else: result = self.connection.Connection_Request() self.is_connected = result return result ## Checks if master and if affirmative, returns the slave # @param target_name The target to connect to. If None gets the name from the default configuration # @return The slave Target or False if not a master target def getSlave(self, target_name=None): if self.is_slave: print "Can't ask for a slave to a slave. Try asking to a master." return False else: # If target is not specified, caller are requesting a "standard" connection to the primary slave target if not target_name: target_name = self.config.getAttr("target_name") slave = Target( True, self.config.getAttr("protocol"), self.config.getAttr("address"), self.config.getAttr("slave_port"), target_name, ) # Sets the session ID slave.config.setAttr("session_id", self.config.session_id) return slave ## Disconnects from the XML-RPC server # @return The boolean result of the operation def disconnect(self): if self.is_slave: result = self.connection.disconnect() self.is_connected = result return result else: print ("Can't call disconnect on master server") return False ## Stops target server. This method, called on a master server, calls connector::Connector::stopServer and called on slave, calls connector::Connection::stopSlave # @return The boolean result of the operation def stop(self): if self.is_slave: # TODO: Stop the data thread if active result = self.connection.stopSlaveServer() self.is_connected = result # Stops data threads self.stop_data_threads() return result else: result = self.connection.stopMasterServer() self.is_connected = result return result ## Closes the session # @return The boolean result of the operation def close(self): result = self.connection.closeSession() # Stops data threads self.stop_data_threads() self.is_connected = result return result ## Stops the target # This method closes the data threads (if they exist) then closes the session # @return The boolean result of the operation def halt(self): self.stop_data_threads() result = self.connection.stopTarget() self.is_connected = result return result ## Starts the connected target # @return The boolean result of the operation def start(self): return self.connection.start() ## Starts data transfer for the specified signal # @param signal_number The number of the signal to start # @param sample_time The sample time of the signal # @param decimation The decimation to use # @return The boolean result of the operation def startData(self, signal_number, decimation=None): sample_time = self.getSampleTimeFromSignalNumber(signal_number) self.setup_data_threads(sample_time, decimation, True) return self.connection.startData(signal_number, decimation) ## Returns sample time given the signal number # @param signal_number The number of the signal to analyze # @return The sample time of the given signal number def getSampleTimeFromSignalNumber(self, signal_number): try: sample_time = None for signal in self.signal_structure: if signal["number"] == signal_number: sample_time = signal["sample_time"] break return sample_time except: return None ## Stops (all) data transfers # @param signal_number The number of signal to catch # @param all_data If True calls StopAllData(). Calls StopData() if False. # @return The boolean result of the operation def stopData(self, signal_number=None, all_data=False): # TODO: Devo chiudere anche i threads dei poller? if all_data: return self.connection.stopAllData() elif signal_number: return self.connection.stopData(signal_number) else: print ("Can't stop data without signal number or all_data flag") return False ## Get the structure of the signals # @return The structure of the signals or False if the target has no signals def getSignalStructure(self): return self.connection.getSignalStructure() ## Sends new parameters to the server # @param params The array of parameters to set # @return The boolean result of the operation def setParameters(self, params): return self.connection.setParameters(params) ## Get the parameters from the target server # @return An array of params. Every param is an hash def getParameters(self): self.parameters = self.connection.getParameters() return self.parameters ## Interface to set a single param # @param identifier The identifier string to get the param to update # @param value The value to set to the param def setParam(self, identifier, value): if not self.parameters: self.parameters = self.connection.getParameters() ids = identifier.split("/") param_name = ids.pop() block_name = "/".join(ids) new_params = [] for param in self.parameters: if param["block_name"] == block_name and param["param_name"] == param_name: param["values"] = value new_params.append(param) self.parameters = new_params return self.setParameters(self.parameters)
def run(self): """Main loop """ self.startCapture() data_collector = DataCollector(self.dataset) modeFullFace = 0 modeOneEye = 1 modePictures = 2 modeTwoEyes = 3 modeImgDB = 4 modeDemo = 5 mode = modeDemo keepLoop = True current_t = time.clock() previous_t = current_t while keepLoop: pressed_key = cv2.waitKey(1) current_t = time.clock() #print('\nclock : ', current_t - previous_t) previous_t = current_t img = self.getCameraImage() if(mode == modeOneEye): ex = 300 ey = 50 eh = 200 ew = 200 face = Face(img.frame, img.canvas, 0, 0, 640, 480) eye = Eye(face.frame, face.canvas, ex, ey, ew, eh) eye.draw(face) eye.iris.normalizeIris() elif(mode == modeTwoEyes): face = Face(img.frame, img.canvas, 0, 0, 640, 480) left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT) left_eye.draw(face) left_eye.iris.normalizeIris() right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT) right_eye.draw(face) right_eye.iris.normalizeIris() elif(mode == modeFullFace): face, left_eye, right_eye = img.detectEyes(self.bufferFace, self.bufferLeftEye, self.bufferRightEye) if face: face.draw(img) if left_eye: left_eye.draw(face) left_eye.iris.normalizeIris() if right_eye: right_eye.draw(face) right_eye.iris.normalizeIris() elif(mode == modeImgDB): img_db = ImageDB("./IR_Database/MMU Iris Database/") print(img_db.estimateUser(img_db.bits[0])) exit() elif(mode == modeDemo): path = "./TB_Database/" face = Face(img.frame, img.canvas, 0, 0, 640, 480) left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT) left_eye.draw(face) left_eye.iris.normalizeIris() right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT) right_eye.draw(face) right_eye.iris.normalizeIris() if(ord('0') <= pressed_key & 0xFF <= ord('9')): print('0-9') id_person = chr(pressed_key & 0xFF) while((pressed_key & 0xFF) not in [ord('a'), ord('p')]): pressed_key = cv2.waitKey(50) print(pressed_key & 0xFF) if(pressed_key & 0xFF == ord('a')): cv2.imwrite(path + id_person + '/left/' + str(int(time.time() * 1000)) + '.bmp', left_eye.frame) cv2.imwrite(path + id_person + '/right/' + str(int(time.time() * 1000)) + '.bmp', right_eye.frame) else: path = "./IR_Database/MMU Iris Database/" for dir in os.listdir(path): subpath = path + dir + '/' if(os.path.isdir(subpath)): print(dir) for subdir in os.listdir(subpath): subsubpath = subpath + subdir + '/' if(os.path.isdir(subsubpath)): print('\t', subdir) for fname in os.listdir(subsubpath): fpath = subsubpath + fname if(os.path.isfile(fpath) and os.path.splitext(fname)[1] == '.bmp'): print('\t\t', fname) img = Image(cv2.imread(fpath)) face = Face(img.frame, img.canvas) eye = Eye(face.frame, face.canvas, padding=10) eye.draw(face) eye.iris.normalizeIris() img.show() pressed_key = cv2.waitKey(1000) exit() # Controls if pressed_key & 0xFF == ord('q'): keepLoop = False #if pressed_key & 0xFF == ord('s'): # self.dataset.save() #if pressed_key & 0xFF == ord('l'): # self.dataset.load() #if pressed_key & 0xFF == ord('m'): # self.showMoments = not self.showMoments #if pressed_key & 0xFF == ord('e'): # self.showEvaluation = not self.showEvaluation #data_collector.step(img.canvas, pressed_key, left_eye, right_eye) #txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset)) #cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) #cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) #if left_eye and right_eye: # direction = self.dataset.estimateDirection(left_eye.computeMomentVectors(), right_eye.computeMomentVectors()) # txt = 'Estimated direction: {}'.format(direction.name) # cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) # cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) img.show() #if self.showEvaluation: # fig = self.dataset.showValidationScoreEvolution() # plt.show() # self.showEvaluation = False #if self.showMoments: # fig = self.dataset.drawVectorizedMoments() # plt.show() # # cv2.imshow('moments', self.fig2cv(fig)) # # plt.close(fig) # self.showMoments = False self.stopCapture()
def collect_data_and_upload(rc=0): """ All the heavy lifting done here Run through "targets" - could be just one (host, default) or many (containers+host) """ # initialize collection targets # for now we do either containers OR host -- not both at same time if InsightsClient.options.container_mode: targets = get_targets() targets = targets + constants.default_target else: targets = constants.default_target if InsightsClient.options.offline: logger.warning("Assuming remote branch and leaf value of -1") pconn = None branch_info = constants.default_branch_info else: pconn = InsightsConnection() # TODO: change these err msgs to be more meaningful , i.e. # "could not determine login information" if pconn: try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information") except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information") pc = InsightsConfig(pconn) if InsightsClient.options.just_upload: if not os.path.exists(InsightsClient.options.just_upload): logger.error('No file %s', InsightsClient.options.just_upload) return 1 tar_file = InsightsClient.options.just_upload rc = _do_upload(pconn, tar_file, 'dummy', 0) return rc # load config from stdin/file if specified try: stdin_config = {} if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) elif InsightsClient.options.from_stdin: stdin_config = json.load(sys.stdin) if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and ('uploader.json' not in stdin_config or 'sig' not in stdin_config)): raise ValueError except: logger.error('ERROR: Invalid config for %s! Exiting...', ('--from-file' if InsightsClient.options.from_file else '--from-stdin')) sys.exit(1) start = time.clock() collection_rules, rm_conf = pc.get_conf(InsightsClient.options.update, stdin_config) collection_elapsed = (time.clock() - start) logger.debug("Rules configuration loaded. Elapsed time: %s", collection_elapsed) individual_archives = [] for t in targets: # defaults archive = None container_connection = None mp = None obfuscate = None # archive metadata archive_meta = {} try: if t['type'] == 'docker_image': container_connection = open_image(t['name']) logging_name = 'Docker image ' + t['name'] archive_meta['display_name'] = get_repotag(t['name']) archive_meta['docker_id'] = t['name'] if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) continue elif t['type'] == 'docker_container': container_connection = open_container(t['name']) logging_name = 'Docker container ' + t['name'] archive_meta['display_name'] = t['name'] if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) continue elif t['type'] == 'host': logging_name = determine_hostname() archive_meta['display_name'] = determine_hostname(InsightsClient.options.display_name) else: logger.error('Unexpected analysis target: %s', t['type']) continue archive_meta['type'] = t['type'].replace('docker_', '') archive_meta['product'] = 'Docker' archive_meta['system_id'] = generate_analysis_target_id(t['type'], t['name']) collection_start = time.clock() archive = InsightsArchive(compressor=InsightsClient.options.compressor if not InsightsClient.options.container_mode else "none", target_name=t['name']) atexit.register(_delete_archive, archive) dc = DataCollector(archive, mountpoint=mp, target_name=t['name'], target_type=t['type']) logger.info('Starting to collect Insights data for %s', logging_name) dc.run_collection(collection_rules, rm_conf, branch_info) elapsed = (time.clock() - start) logger.debug("Data collection complete. Elapsed time: %s", elapsed) obfuscate = InsightsClient.config.getboolean(APP_NAME, "obfuscate") # include rule refresh time in the duration collection_duration = (time.clock() - collection_start) + collection_elapsed if InsightsClient.options.no_tar_file: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc tar_file = dc.done(collection_rules, rm_conf) # add archives to list of individual uploads archive_meta['tar_file'] = tar_file individual_archives.append(archive_meta) finally: # called on loop iter end or unexpected exit if container_connection: container_connection.close() # if multiple targets (container mode), add all archives to single archive if InsightsClient.options.container_mode: full_archive = InsightsArchive(compressor=InsightsClient.options.compressor) for a in individual_archives: shutil.copy(a['tar_file'], full_archive.archive_dir) # don't want insights_commands in meta archive shutil.rmtree(full_archive.cmd_dir) metadata = _create_metadata_json(individual_archives) full_archive.add_metadata_to_archive(json.dumps(metadata), 'metadata.json') full_tar_file = full_archive.create_tar_file(full_archive=True) # if only one target (regular mode), just upload one else: full_archive = archive full_tar_file = tar_file if InsightsClient.options.offline or InsightsClient.options.no_upload: handle_file_output(full_tar_file, full_archive) return rc # do the upload rc = _do_upload(pconn, full_tar_file, logging_name, collection_duration) if InsightsClient.options.keep_archive: logger.info('Insights data retained in %s', full_tar_file) return rc if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(full_tar_file)) full_archive.delete_archive_dir() return rc
def __init__(self): self.data_collector = DataCollector() self.new_location = DataCollector()
class Monitor: def __init__(self): self.data_collector = DataCollector() self.new_location = DataCollector() def collect_default_data(self): self.data_collector.filter_radius() def relocate(self, location=None, coordinates=None, radius: int = 250): if location is not None: geolocator = Nominatim(user_agent="team_1_earthquake_detector") location_latitude = geolocator.geocode(location).latitude location_longitude = geolocator.geocode(location).longitude location_coordinates = (location_latitude, location_longitude) return self.new_location.filter_radius(location_coordinates, radius) if coordinates is not None: return self.new_location.filter_radius(coordinates, radius) def build_map(self, location=None, coordinates=None, radius=None): self.map = Map() if location is None and coordinates is None: self.map.set_up_map() self.collect_default_data() earthquake_overlay = EarthquakeOverlay( self.data_collector.earthquake_data_clean) earthquake_overlay.apply_circle_markers(self.map.map) earthquake_overlay.apply_magnitude_markers(self.map.map) earthquake_overlay.apply_connective_lines(self.map.map) earthquake_overlay.apply_heatmap(self.map.map) tectonic_overlay = TectonicOverlay() tectonic_overlay.apply_overlay(self.map.map) tectonic_overlay.add_to_layer_control(self.map.map) return self.map if location is not None or coordinates is not None: self.map.set_up_map(location=coordinates) self.relocate(location=location, coordinates=coordinates, radius=radius) earthquake_overlay = EarthquakeOverlay( self.new_location.earthquake_data_clean) earthquake_overlay.apply_circle_markers(self.map.map) earthquake_overlay.apply_magnitude_markers(self.map.map) earthquake_overlay.apply_connective_lines(self.map.map, coordinates) earthquake_overlay.apply_heatmap(self.map.map) tectonic_overlay = TectonicOverlay() tectonic_overlay.apply_overlay(self.map.map) tectonic_overlay.add_to_layer_control(self.map.map) return self.map def perform_earthquake_analytics(self, location=None, radius=None): self.data_collector.prep_data() self.data_collector.filter_radius() earthquake_analytics = EarthquakeAnalytics( self.data_collector.earthquake_data, self.data_collector.earthquake_data_clean) total_filtered = earthquake_analytics.get_total_filtered_earthquakes( location=location, radius=radius) minor_filtered = earthquake_analytics.get_filtered_minor_earthquakes( location=location, radius=radius) moderate_filtered = earthquake_analytics.get_filtered_moderate_earthquakes( location=location, radius=radius) strong_filtered = earthquake_analytics.get_filtered_strong_earthquakes( location=location, radius=radius) closest_filtered = earthquake_analytics.get_closest_filtered_earthquake( location=location) place_of_closest_filtered = earthquake_analytics.get_place_of_closest_filtered_earthquake( location=location) strongest_filtered = earthquake_analytics.get_strongest_filtered_earthquake( location=location, radius=radius) total_worldwide = earthquake_analytics.get_total_earthquakes_worldwide( ) minor_worldwide = earthquake_analytics.get_minor_earthquakes_worldwide( ) moderate_worldwide = earthquake_analytics.get_moderate_earthquakes_worldwide( ) strong_worldwide = earthquake_analytics.get_strong_earthquakes_worldwide( ) strongest_worldwide = earthquake_analytics.get_strongest_earthquake_worldwide( ) place_of_strongest_worldwide = earthquake_analytics.get_place_of_strongest_earthquake_worldwide( ) return total_filtered, minor_filtered, moderate_filtered, strong_filtered, closest_filtered, place_of_closest_filtered,\ strongest_filtered, total_worldwide, minor_worldwide, moderate_worldwide, strong_worldwide, strongest_worldwide, \ place_of_strongest_worldwide
def run(self): """Main loop """ self.startCapture() data_collector = DataCollector(self.dataset) keepLoop = True while keepLoop: pressed_key = cv2.waitKey(1) img = self.getCameraImage() face, left_eye, right_eye = img.detectEyes(self.bufferFace, self.bufferLeftEye, self.bufferRightEye) if face: face.draw(img) if left_eye: left_eye.draw(face) if right_eye: right_eye.draw(face) # Controls if pressed_key & 0xFF == ord('q'): keepLoop = False if pressed_key & 0xFF == ord('s'): self.dataset.save() if pressed_key & 0xFF == ord('l'): self.dataset.load() if pressed_key & 0xFF == ord('m'): self.showMoments = not self.showMoments if pressed_key & 0xFF == ord('e'): self.showEvaluation = not self.showEvaluation data_collector.step(img.canvas, pressed_key, left_eye, right_eye) txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset)) cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) if left_eye and right_eye: direction = self.dataset.estimateDirection( left_eye.computeMomentVectors(), right_eye.computeMomentVectors()) txt = 'Estimated direction: {}'.format(direction.name) cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2) cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2) img.show() if self.showEvaluation: fig = self.dataset.showValidationScoreEvolution() plt.show() self.showEvaluation = False if self.showMoments: fig = self.dataset.drawVectorizedMoments() plt.show() # cv2.imshow('moments', self.fig2cv(fig)) # plt.close(fig) self.showMoments = False self.stopCapture()
def get_node_stats(node_name): node = NodeRepository.by_name(node_name) data = DataCollector.get_node_info(node) StatsRepository.save_for_node(node, data)
parser.add_option("-k", action="store", dest="k", type="long", default=3) parser.add_option("--temp", action="store", dest="temp_rate", type="float", default=0.9) parser.add_option("--iters", action="store", dest="iters", type="float", default=100) parser.add_option("--max_t", action="store", dest="max_temp", type="float", default=100) parser.add_option("--min_t", action="store", dest="min_temp", type="float", default=0.5) parser.add_option("--hh", action="store", type="string", dest="hh") global options (options, _) = parser.parse_args() # data collector configuration global collector collector = DataCollector()
def __init__(self, conf): DataCollector.__init__(self) self.conf = conf