def testRunWithError(self):
collector = DataCollector(in_path=IN_FILE_BAD,
ot_path_data=OT_FILE_DATA,
ot_path_doc=OT_FILE_DOC)
collector.run()
df = pd.read_csv(OT_FILE_DATA)
self.assertEqual(len(df["Biomodel_Id"]), 3)
def run_random_search(verbose,
num_diff_experiments,
num_repeat_experiment,
allow_duplicates=False,
df_path=None,
overwrite=True,
data_to_collect=POSSIBLE_DATA,
MVP_key='waitingTime',
save_model=True):
grid = load_constants('constants/constants-grid.json')
if not allow_duplicates:
_, num_choices = get_num_grid_choices(grid)
num_diff_experiments = min(num_choices, num_diff_experiments)
# Make grid choice generator
grid_choice_gen = grid_choices_random(grid, num_diff_experiments)
for diff_experiment, constants in enumerate(grid_choice_gen):
data_collector_obj = DataCollector(
data_to_collect, MVP_key, constants,
'test' if constants['agent']['agent_type'] == 'rule' else 'eval',
df_path, overwrite if diff_experiment == 0 else False, verbose)
for same_experiment in range(num_repeat_experiment):
print(' --- Running experiment {}.{} / {}.{} --- '.format(
diff_experiment + 1, same_experiment + 1, num_diff_experiments,
num_repeat_experiment))
if save_model:
data_collector_obj.set_save_model_path(
'models/saved_models/random_{}-{}.pt'.format(
diff_experiment + 1, same_experiment + 1))
run_experiment(diff_experiment + 1, same_experiment + 1, constants,
data_collector_obj)
def setup_data_threads(self, sample_time = None, decimation = None, start_threads = False):
if self.is_slave:
if not self.datastream_queue:
self.datastream_queue = []
# TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller.
if not self.datastream_poller:
self.datastream_poller = DataPoller(self.datastream_queue, decimation)
# TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati
if not self.datastream_collector:
# TODO: Fermare i thread alla chiusura del target
self.datastream_collector = DataCollector(self.config.getAttr('address'), self.config.getAttr('slave_data_port'), self.datastream_queue)
if start_threads:
print("Starting data threads...")
if sample_time:
self.datastream_poller.updateSampleTime(sample_time)
self.datastream_poller.start()
self.datastream_collector.start()
print("Data threads started...")
else:
print("Can't start data threads without a sample time")
return False
return True
else:
print("Can't setup the data stream threads on a not-slave server")
return False
def start(self):
with self.__lock:
self.__need_to_stop_flag = False
self.__run_browser()
once_authorized = False
self.__load_url(YandexWordstatParser.BASE_URL)
for phrase in self.__phrases:
self.__type_in_phrase_to_search_field_and_press_find(phrase)
if not once_authorized:
try:
self.__authorize_on_redirected_form()
except TimeoutException:
self.__authorize()
once_authorized = True
# wait until DOM is not built to avoid selenium.common.exceptions.StaleElementReferenceException
time.sleep(1)
DataCollector.store_wordstat_data(self.__extract_keywords(),
phrase)
YandexWordstatParser.__random_wait(3000, 15000)
self.__driver.quit()
self.on_completed.emit()
def main():
args = get_args()
data_collector_obj = DataCollector(args.debug)
if args.debug:
print("Got arguments " + str(args.years) + ", " + str(args.players) +
", " + args.mode + ", " + str(args.singles))
data_collector_obj.scrape_last_n_years_of_k_players(
args.years, args.players, args.mode,
"singles" if args.singles else "doubles")
def setUp(self):
data_collector = DataCollector(config=config)
self.data_after = data_collector.data_collector()
path = 'devlab/tests'
file_name = config.rollback_params['data_file_names']['PRE']
pre_file_path = os.path.join(data_collector.main_folder, path,
file_name)
with open(pre_file_path, "r") as f:
self.pre_data = yaml.load(f)
def setUp(self):
data_collector = DataCollector(config=config)
self.data_after = data_collector.data_collector()
path = 'devlab/tests'
file_name = config.rollback_params['data_file_names']['PRE']
pre_file_path = os.path.join(data_collector.main_folder, path,
file_name)
with open(pre_file_path, "r") as f:
self.pre_data = yaml.load(f)
def start(self):
with self.__lock:
self.__need_to_stop_flag = False
self.__run_browser()
for phrase in self.__phrases:
result = []
for city_code in self.__city_codes:
if self.__need_to_stop():
break
variables = {
YandexParser.TEXT_VARIABLE_NAME: phrase,
YandexParser.CITY_CODE_VARIABLE_NAME: city_code
}
self.__load_url(YandexParser.BASE_URL + '?' +
urllib.parse.urlencode(variables))
def parse_ads_city_phrase_loop():
nonlocal result
while len(result) < self.__ads_count_for_one_key:
result += self.__parse_loaded_page(
self.__ads_count_for_one_key - len(result))
if len(result) >= self.__ads_count_for_one_key:
break
else:
try:
next_page_link = self.__driver.find_element_by_class_name(
'pager__item_kind_next')
time.sleep(self.__pause_time_between_requests /
1000)
self.__driver.execute_script('console.clear()')
next_page_link.click()
except NoSuchElementException as error:
self.__logger.warning(
'There is no more ads for phrase \'%s\'' %
phrase)
self.__logger.warning(error)
break
parse_ads_city_phrase_loop()
self.__logger.info(result)
result = self._remove_duplicated_hosts(result)
DataCollector.store_ad_search_data('YandexParser', phrase, result)
self.__driver.quit()
self.on_completed.emit()
def __init__(self, api_host, gatsby_host, alias_host, num_threads):
self.select_clause = "mpidStr AS \'mpid\', priceRange, aggregatedRatings, modelTitle AS \'title\', brandName, categoryNamePath, searchScore, brandName, storeId, image"
self.page_size = 500
self.country_code = 356
self.gatsby_query = ProductGatsbyQuery(self.select_clause,
self.page_size,
self.country_code,
"/products/search2",
gatsby_host)
self.gatsbyPB_query = ProductGatsbyQuery(self.select_clause,
self.page_size,
self.country_code,
"/products/search2",
gatsby_host)
self.api_query = ProductApiQuery("http", api_host, "/v2.1/search",
"IN", 50)
self.alias_service = DataCollector(
ProductAliasQuery("http", alias_host, "/search", "IN"))
self.ranking_model = RankingModel()
self.gatsby_queries = Queue(2)
self.gatsbyPB_queries = Queue(2)
self.api_queries = Queue(2)
worker = Worker(DataCollector(self.gatsby_query), self.ranking_model,
self.gatsby_queries)
# worker.daemon = True
print "currently running thread:\t", worker.getName()
worker.start()
worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model,
self.gatsbyPB_queries)
print "currently running thread:\t", worker.getName()
# worker.daemon = True
worker.start()
# for thread in range(2):
# worker = Worker(DataCollector(self.gatsbyPB_query), self.ranking_model, self.gatsbyPB_queries)
# print "currently running thread:\t", worker.getName()
# worker.daemon = True
# worker.start()
#
# for thread in range(2):
# worker = Worker(DataCollector(self.gatsby_query), self.ranking_model, self.gatsby_queries)
# worker.daemon = True
# worker.start()
worker = Worker(DataCollector(self.api_query), self.ranking_model,
self.api_queries)
print "currently running thread:\t", worker.getName()
# worker.daemon = True
worker.start()
def setup_data_threads(self, sample_time=None, decimation=None, start_threads=False):
if self.is_slave:
if not self.datastream_queue:
self.datastream_queue = []
# TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller.
if not self.datastream_poller:
self.datastream_poller = DataPoller(self.datastream_queue, decimation)
# TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati
if not self.datastream_collector:
# TODO: Fermare i thread alla chiusura del target
self.datastream_collector = DataCollector(
self.config.getAttr("address"), self.config.getAttr("slave_data_port"), self.datastream_queue
)
if start_threads:
print ("Starting data threads...")
if sample_time:
self.datastream_poller.updateSampleTime(sample_time)
self.datastream_poller.start()
self.datastream_collector.start()
print ("Data threads started...")
else:
print ("Can't start data threads without a sample time")
return False
return True
else:
print ("Can't setup the data stream threads on a not-slave server")
return False
def register(config, group_id=None):
"""
Do registration using basic auth
"""
username = config.get(APP_NAME, 'username')
password = config.get(APP_NAME, 'password')
if (((username == "") and
(password == "") and
(config.get(APP_NAME, 'authmethod') == 'BASIC')) and not
(config.get(APP_NAME, 'auto_config'))):
# Get input from user
print "Please enter your Red Hat Customer Portal Credentials"
sys.stdout.write('User Name: ')
username = raw_input().strip()
password = getpass.getpass()
sys.stdout.write("Would you like to save these credentials? (y/n) ")
save = raw_input().strip()
config.set(APP_NAME, 'username', username)
config.set(APP_NAME, 'password', password)
logger.debug("savestr: %s", save)
if save.lower() == "y" or save.lower() == "yes":
logger.debug("writing user/pass to config file")
cmd = ("/bin/sed -e 's/^username.*=.*$/username="******"/' " +
"-e 's/^password.*=.*$/password="******"/' " +
constants.default_conf_file)
status = DataCollector().run_command_get_output(cmd, nolog=True)
config_file = open(constants.default_conf_file, 'w')
config_file.write(status['output'])
config_file.flush()
pconn = InsightsConnection(config)
return pconn.register(group_id)
def __init__(self, data_collector=None): if data_collector is None: data_collector = DataCollector() else: self.data_collector = data_collector self.bigrams = dict() self.get_unique_bigrams()
def mean_std_of_states():
env = MountainCarWithResetEnv()
samples_to_collect = 100000
states, actions, rewards, next_states, done_flags = DataCollector(env).collect_data(samples_to_collect)
all_states = np.concatenate((states, next_states))
states_mean = np.mean(all_states, axis=0)
states_std = np.std(all_states, axis=0)
print("states_mean: {}, states_std: {}".format(states_mean, states_std))
class Preprocessing:
def __init__(self):
self.di = DataCollector()
def to_upper(self):
data = self.di.collect_data()
upper = data['data'].upper()
return upper
def __init__(self):
with DataCollector() as collector:
result_channels = collector.get_telemetry_outputs()
with DataServer(result_channels) as transmitter:
print("Press ctrl+c to exit")
signal.signal(signal.SIGINT, self.exit_handler)
# TODO signal.pause() is not available on Windows systems
signal.pause()
def perfrom_experiment(self, experiment_name='test', movement_list=[]):
# 1. We save the background image:
dc = DataCollector(only_one_shot=False, automatic=True)
dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=False, save=True, directory=experiment_name+'/air', iteration=-1)
print "Air data gathered"
# 2. We perfomr the experiment:
i = 0
if not os.path.exists(experiment_name): # If the directory does not exist, we create it
os.makedirs(experiment_name)
for movement in movement_list:
path = experiment_name + '/p_' + str(i) + '/'
self.palpate(speed=40, force_list=self.force_list, save=True, path=path)
self.move_cart_mm(movement[0], movement[1], movement[2])
time.sleep(6)
i += 1
path = experiment_name + '/p_' + str(i) + '/'
self.palpate(speed=40, force_list=self.force_list, save=True, path=path)
def training_the_model(samples_to_collect=100000, seed=100):
number_of_kernels_per_dim = [10, 8]
gamma = 0.999
w_updates = 20
evaluation_number_of_games = 50
evaluation_max_steps_per_game = 300
np.random.seed(seed)
env = MountainCarWithResetEnv()
# collect data
states, actions, rewards, next_states, done_flags = DataCollector(
env).collect_data(samples_to_collect)
# get data success rate
data_success_rate = np.sum(rewards) / len(rewards)
print(f'Data Success Rate {data_success_rate}')
# standardize data
data_transformer = DataTransformer()
data_transformer.set_using_states(
np.concatenate((states, next_states), axis=0))
states = data_transformer.transform_states(states)
next_states = data_transformer.transform_states(next_states)
# process with radial basis functions
feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim)
# encode all states:
encoded_states = feature_extractor.encode_states_with_radial_basis_functions(
states)
encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions(
next_states)
# set a new linear policy
linear_policy = LinearPolicy(feature_extractor.get_number_of_features(), 3,
True)
# but set the weights as random
linear_policy.set_w(np.random.uniform(size=linear_policy.w.shape))
# start an object that evaluates the success rate over time
evaluator = GamePlayer(env, data_transformer, feature_extractor,
linear_policy)
success_rate_vs_iteration = list()
for lspi_iteration in range(w_updates):
print(f'Starting LSPI iteration {lspi_iteration}')
new_w = compute_lspi_iteration(encoded_states, encoded_next_states,
actions, rewards, done_flags,
linear_policy, gamma)
norm_diff = linear_policy.set_w(new_w)
success_rate = evaluator.play_games(evaluation_number_of_games,
evaluation_max_steps_per_game)
success_rate_vs_iteration.append(success_rate)
if norm_diff < 0.00001:
break
print('LSPI Done')
return success_rate_vs_iteration
def palpate(self, speed=40, force_list=[1, 10, 20, 40], save=False, path=''):
# 0. We create the directory
if save is True and not os.path.exists(path): # If the directory does not exist, we create it
os.makedirs(path)
# 1. We get and save the cartesian coord.
dc = DataCollector(only_one_shot=False, automatic=True)
cart = dc.getCart()
if save is True:
np.save(path + '/cart.npy', cart)
# 2. We get wsg forces and gs images at every set force and store them
i = 0
for force in force_list:
self.close_gripper_f(grasp_speed=speed, grasp_force=force)
print "Applying: " + str(force)
time.sleep(1.0)
dc.get_data(get_cart=False, get_gs1=(1 in self.gs_id), get_gs2=(2 in self.gs_id), get_wsg=True, save=save, directory=path, iteration=i)
self.open_gripper()
time.sleep(1.0)
i += 1
def __init__(self):
self.screen = MeteoScreen()
self.collector = DataCollector()
self.data = InMemorySensorDataLogger()
self.collector.add_logger(self.data)
file_logger = ToFileSensorDataLogger('./db')
self.collector.add_logger(file_logger)
now = datetime.datetime.now()
last24h_data = file_logger.load_data(now - datetime.timedelta(days=1),
now)
self.data.set_data(last24h_data)
self.screen_update_interval = 10
self.current_screen_draw = self.draw_screen1
self.update_timer_stop = threading.Event()
self.screen_update_lock = threading.RLock()
def timer():
while not self.update_timer_stop.is_set():
time.sleep(self.screen_update_interval)
self.update_screen()
self.timer_thread = threading.Thread(target=timer)
class TestDataCollector(unittest.TestCase):
def setUp(self):
self.collector = DataCollector(in_path=IN_FILE,
ot_path_data=OT_FILE_DATA,
ot_path_doc=OT_FILE_DOC)
def testConstructor(self):
if IGNORE_TEST:
return
self.assertEqual(self.collector._ot_path_data, OT_FILE_DATA)
def testRun(self):
self.collector.run()
df = pd.read_csv(OT_FILE_DATA)
self.assertEqual(len(df["Biomodel_Id"]), 2)
def testRunWithError(self):
collector = DataCollector(in_path=IN_FILE_BAD,
ot_path_data=OT_FILE_DATA,
ot_path_doc=OT_FILE_DOC)
collector.run()
df = pd.read_csv(OT_FILE_DATA)
self.assertEqual(len(df["Biomodel_Id"]), 3)
def run_normal(verbose,
num_experiments=1,
df_path=None,
overwrite=True,
data_to_collect=POSSIBLE_DATA,
MVP_key='waitingTime',
save_model=True,
load_model_file=None):
# if loading, then dont save
if load_model_file:
save_model = False
if not df_path:
df_path = 'run-data.xlsx' # def. path
# Load constants
constants = load_constants('constants/constants.json')
data_collector_obj = DataCollector(
data_to_collect, MVP_key, constants,
'test' if constants['agent']['agent_type'] == 'rule' or load_model_file
else 'eval', df_path, overwrite, verbose)
loaded_model = None
if load_model_file:
loaded_model = torch.load('models/saved_models/' + load_model_file)
for exp in range(num_experiments):
print(' --- Running experiment {} / {} --- '.format(
exp + 1, num_experiments))
if save_model:
data_collector_obj.set_save_model_path(
'models/saved_models/normal_{}.pt'.format(exp + 1))
run_experiment(exp + 1,
None,
constants,
data_collector_obj,
loaded_model=loaded_model)
def collect_data_and_upload(config, options):
"""
All the heavy lifting done here
"""
pconn = InsightsConnection(config)
pconn.check_registration()
branch_info = pconn.branch_info()
pc = InsightsConfig(config, pconn)
dc = DataCollector()
logger.info('Collecting Insights data')
start = time.clock()
dynamic_config = pc.get_conf(options.update)
elapsed = (time.clock() - start)
logger.debug("Dynamic Config Elapsed Time: %s", elapsed)
start = time.clock()
dc.run_commands(dynamic_config)
elapsed = (time.clock() - start)
logger.debug("Command Collection Elapsed Time: %s", elapsed)
start = time.clock()
dc.copy_files(dynamic_config)
elapsed = (time.clock() - start)
logger.debug("File Collection Elapsed Time: %s", elapsed)
dc.write_branch_info(branch_info)
obfuscate = config.getboolean(APP_NAME, "obfuscate")
if not options.no_tar_file:
tar_file = dc.done(config)
if not options.no_upload:
logger.info('Uploading Insights data,'
' this may take a few minutes')
pconn.upload_archive(tar_file)
logger.info(
'Check https://access.redhat.com/labs/insights in an hour')
if not obfuscate and not options.keep_archive:
dc.archive.delete_tmp_dir()
else:
if obfuscate:
logger.info('Obfuscated Insights data retained in %s',
os.path.dirname(tar_file))
else:
logger.info('Insights data retained in %s', tar_file)
else:
logger.info('See Insights data in %s', tar_file)
else:
logger.info('See Insights data in %s', dc.archive.archive_dir)
if __name__ == '__main__':
samples_to_collect = 100000
# samples_to_collect = 150000
# samples_to_collect = 10000
number_of_kernels_per_dim = [10, 8]
gamma = 0.99
w_updates = 100
evaluation_number_of_games = 10
evaluation_max_steps_per_game = 1000
np.random.seed(123)
# np.random.seed(234)
env = MountainCarWithResetEnv()
# collect data
states, actions, rewards, next_states, done_flags = DataCollector(
env).collect_data(samples_to_collect)
# get data success rate
data_success_rate = np.sum(rewards) / len(rewards)
print(f'success rate {data_success_rate}')
# standardize data
data_transformer = DataTransformer()
data_transformer.set_using_states(
np.concatenate((states, next_states), axis=0))
states = data_transformer.transform_states(states)
next_states = data_transformer.transform_states(next_states)
# process with radial basis functions
feature_extractor = RadialBasisFunctionExtractor(number_of_kernels_per_dim)
# encode all states:
encoded_states = feature_extractor.encode_states_with_radial_basis_functions(
states)
encoded_next_states = feature_extractor.encode_states_with_radial_basis_functions(
#!/usr/bin/python
from data_collector import DataCollector
import rospy
import sys
if __name__ == "__main__":
rospy.init_node("data_collector")
collector = DataCollector()
rospy.spin()
collector.dump()
def test_makes_get_request_to_node_fullpath(self, request):
some_path = "http://localhost:1231"
node = Mock(**{"full_path.return_value": some_path})
data = DataCollector.get_node_info(node)
request.assert_called_with(some_path)
def test_gets_data_from_node(self, node_data):
data = DataCollector.get_node_info(Mock())
assert_that(data, is_(SAMPLE_NODE_DATA))
def collect_data_and_upload(config, options, rc=0):
"""
All the heavy lifting done here
"""
collection_start = time.clock()
pconn = InsightsConnection(config)
try:
branch_info = pconn.branch_info()
except requests.ConnectionError:
branch_info = handle_branch_info_error(
"Could not connect to determine branch information", options)
except LookupError:
branch_info = handle_branch_info_error(
"Could not determine branch information", options)
pc = InsightsConfig(config, pconn)
archive = InsightsArchive(compressor=options.compressor)
dc = DataCollector(archive)
# register the exit handler here to delete the archive
atexit.register(handle_exit, archive, options.keep_archive or options.no_upload)
stdin_config = json.load(sys.stdin) if options.from_stdin else {}
start = time.clock()
collection_rules, rm_conf = pc.get_conf(options.update, stdin_config)
elapsed = (time.clock() - start)
logger.debug("Collection Rules Elapsed Time: %s", elapsed)
start = time.clock()
logger.info('Starting to collect Insights data')
dc.run_commands(collection_rules, rm_conf)
elapsed = (time.clock() - start)
logger.debug("Command Collection Elapsed Time: %s", elapsed)
start = time.clock()
dc.copy_files(collection_rules, rm_conf)
elapsed = (time.clock() - start)
logger.debug("File Collection Elapsed Time: %s", elapsed)
dc.write_branch_info(branch_info)
obfuscate = config.getboolean(APP_NAME, "obfuscate")
collection_duration = (time.clock() - collection_start)
if not options.no_tar_file:
tar_file = dc.done(config, rm_conf)
if not options.no_upload:
logger.info('Uploading Insights data,'
' this may take a few minutes')
for tries in range(options.retries):
upload = pconn.upload_archive(tar_file, collection_duration)
if upload.status_code == 201:
write_lastupload_file()
logger.info("Upload completed successfully!")
break
elif upload.status_code == 412:
pconn.handle_fail_rcs(upload)
else:
logger.error("Upload attempt %d of %d failed! Status Code: %s",
tries + 1, options.retries, upload.status_code)
if tries + 1 != options.retries:
logger.info("Waiting %d seconds then retrying",
constants.sleep_time)
time.sleep(constants.sleep_time)
else:
logger.error("All attempts to upload have failed!")
logger.error("Please see %s for additional information",
constants.default_log_file)
rc = 1
if (not obfuscate and not options.keep_archive) or options.no_upload:
dc.archive.delete_tmp_dir()
else:
if obfuscate:
logger.info('Obfuscated Insights data retained in %s',
os.path.dirname(tar_file))
else:
logger.info('Insights data retained in %s', tar_file)
else:
handle_file_output(options, tar_file)
else:
logger.info('See Insights data in %s', dc.archive.archive_dir)
return rc
#maybe?
'''
def get_possible_seed_words(self, length):
all_tokens = get_unique_tokens_count(self.data_collector.corpus)
possible_seeds = []
for tkn in all_tokens:
if (len(tkn[0]) == int(length)):
possible_seeds.append(tkn[0])
return possible_seeds'''
#collect data/clean it
collector = DataCollector()
collector.clean_corpus()
#print(collector.corpus)
bigram_model = BigramModel(collector)
dir_name = 'txt_examples'
for filename in os.listdir(dir_name):
print('INFO: generating text for {fname}'.format(fname=filename))
word_lengths = []
f = open('{directory}/{file}'.format(directory=dir_name, file=filename), 'r')
for line in f:
pos = tuple(frame.position[:3])
rot = tfx.tb_angles(frame.rotation)
rot = (rot.yaw_deg, rot.pitch_deg, rot.roll_deg)
pickle.dump({'pos': pos, 'rot': rot, 'estimate': estimate}, f)
f.close()
psm1 = robot("PSM1")
psm1.open_gripper(80)
time.sleep(2)
d = DataCollector()
time.sleep(2)
img = cv2.medianBlur(d.left_image[:, 850:], 13)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = cv2.bilateralFilter(img, 11, 17, 17)
output = cv2.Canny(img, 100, 200)
(cnts, _) = cv2.findContours(output.copy(), cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)
for c in cnts:
# approximate the contour
peri = cv2.arcLength(c, True)
def collect_data_and_upload(config, options):
"""
All the heavy lifting done here
"""
pconn = InsightsConnection(config)
try:
branch_info = pconn.branch_info()
except requests.ConnectionError:
logger.error("ERROR: Could not connect to determine branch information")
sys.exit()
except LookupError:
logger.error("ERROR: Could not determine branch information")
sys.exit()
pc = InsightsConfig(config, pconn)
dc = DataCollector()
start = time.clock()
collection_rules, rm_conf = pc.get_conf(options.update)
elapsed = (time.clock() - start)
logger.debug("Collection Rules Elapsed Time: %s", elapsed)
start = time.clock()
logger.info('Starting to collect Insights data')
dc.run_commands(collection_rules, rm_conf)
elapsed = (time.clock() - start)
logger.debug("Command Collection Elapsed Time: %s", elapsed)
start = time.clock()
dc.copy_files(collection_rules, rm_conf)
elapsed = (time.clock() - start)
logger.debug("File Collection Elapsed Time: %s", elapsed)
dc.write_branch_info(branch_info)
obfuscate = config.getboolean(APP_NAME, "obfuscate")
if not options.no_tar_file:
tar_file = dc.done(config, rm_conf)
if not options.no_upload:
logger.info('Uploading Insights data,'
' this may take a few minutes')
for tries in range(options.retries):
status = pconn.upload_archive(tar_file)
if status == 201:
logger.info("Upload completed successfully!")
break
else:
logger.error("Upload attempt %d of %d failed! Status Code: %s",
tries+1, options.retries, status)
if tries +1 != options.retries:
logger.info("Waiting %d seconds then retrying", constants.sleep_time)
time.sleep(constants.sleep_time)
else:
logger.error("All attempts to upload have failed!")
logger.error("Please see %s for additional information", constants.default_log_file)
if not obfuscate and not options.keep_archive:
dc.archive.delete_tmp_dir()
else:
if obfuscate:
logger.info('Obfuscated Insights data retained in %s',
os.path.dirname(tar_file))
else:
logger.info('Insights data retained in %s', tar_file)
else:
logger.info('See Insights data in %s', tar_file)
else:
logger.info('See Insights data in %s', dc.archive.archive_dir)
def collect(self, dir):
DataCollector.collect(self, dir)
try:
self.total_authors = int(getpipeoutput(["git log", "git shortlog -s", "wc -l"]))
except:
self.total_authors = 0
# self.total_lines = int(getoutput('git-ls-files -z |xargs -0 cat |wc -l'))
self.activity_by_hour_of_day = {} # hour -> commits
self.activity_by_day_of_week = {} # day -> commits
self.activity_by_month_of_year = {} # month [1-12] -> commits
self.activity_by_hour_of_week = {} # weekday -> hour -> commits
self.activity_by_hour_of_day_busiest = 0
self.activity_by_hour_of_week_busiest = 0
self.activity_by_year_week = {} # yy_wNN -> commits
self.activity_by_year_week_peak = 0
self.authors = (
{}
) # name -> {commits, first_commit_stamp, last_commit_stamp, last_active_day, active_days, lines_added, lines_removed}
# domains
self.domains = {} # domain -> commits
# author of the month
self.author_of_month = {} # month -> author -> commits
self.author_of_year = {} # year -> author -> commits
self.commits_by_month = {} # month -> commits
self.commits_by_year = {} # year -> commits
self.first_commit_stamp = 0
self.last_commit_stamp = 0
self.last_active_day = None
self.active_days = set()
# lines
self.total_lines = 0
self.total_lines_added = 0
self.total_lines_removed = 0
# timezone
self.commits_by_timezone = {} # timezone -> commits
# tags
self.tags = {}
lines = getpipeoutput(["git show-ref --tags"]).split("\n")
for line in lines:
if len(line) == 0:
continue
(hash, tag) = line.split(" ")
tag = tag.replace("refs/tags/", "")
output = getpipeoutput(['git log "%s" --pretty=format:"%%at %%an" -n 1' % hash])
if len(output) > 0:
parts = output.split(" ")
stamp = 0
try:
stamp = int(parts[0])
except ValueError:
stamp = 0
self.tags[tag] = {
"stamp": stamp,
"hash": hash,
"date": datetime.datetime.fromtimestamp(stamp).strftime("%Y-%m-%d"),
"commits": 0,
"authors": {},
}
# collect info on tags, starting from latest
tags_sorted_by_date_desc = map(
lambda el: el[1], reversed(sorted(map(lambda el: (el[1]["date"], el[0]), self.tags.items())))
)
prev = None
for tag in reversed(tags_sorted_by_date_desc):
cmd = 'git shortlog -s "%s"' % tag
if prev != None:
cmd += ' "^%s"' % prev
output = getpipeoutput([cmd])
if len(output) == 0:
continue
prev = tag
for line in output.split("\n"):
parts = re.split("\s+", line, 2)
commits = int(parts[1])
author = parts[2]
self.tags[tag]["commits"] += commits
self.tags[tag]["authors"][author] = commits
# Collect revision statistics
# Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
lines = getpipeoutput(['git rev-list --pretty=format:"%at %ai %an <%aE>" HEAD', "grep -v ^commit"]).split("\n")
for line in lines:
parts = line.split(" ", 4)
author = ""
try:
stamp = int(parts[0])
except ValueError:
stamp = 0
timezone = parts[3]
author, mail = parts[4].split("<", 1)
author = author.rstrip()
mail = mail.rstrip(">")
domain = "?"
if mail.find("@") != -1:
domain = mail.rsplit("@", 1)[1]
date = datetime.datetime.fromtimestamp(float(stamp))
# First and last commit stamp
if self.last_commit_stamp == 0:
self.last_commit_stamp = stamp
self.first_commit_stamp = stamp
# activity
# hour
hour = date.hour
self.activity_by_hour_of_day[hour] = self.activity_by_hour_of_day.get(hour, 0) + 1
# most active hour?
if self.activity_by_hour_of_day[hour] > self.activity_by_hour_of_day_busiest:
self.activity_by_hour_of_day_busiest = self.activity_by_hour_of_day[hour]
# day of week
day = date.weekday()
self.activity_by_day_of_week[day] = self.activity_by_day_of_week.get(day, 0) + 1
# domain stats
if domain not in self.domains:
self.domains[domain] = {}
# commits
self.domains[domain]["commits"] = self.domains[domain].get("commits", 0) + 1
# hour of week
if day not in self.activity_by_hour_of_week:
self.activity_by_hour_of_week[day] = {}
self.activity_by_hour_of_week[day][hour] = self.activity_by_hour_of_week[day].get(hour, 0) + 1
# most active hour?
if self.activity_by_hour_of_week[day][hour] > self.activity_by_hour_of_week_busiest:
self.activity_by_hour_of_week_busiest = self.activity_by_hour_of_week[day][hour]
# month of year
month = date.month
self.activity_by_month_of_year[month] = self.activity_by_month_of_year.get(month, 0) + 1
# yearly/weekly activity
yyw = date.strftime("%Y-%W")
self.activity_by_year_week[yyw] = self.activity_by_year_week.get(yyw, 0) + 1
if self.activity_by_year_week_peak < self.activity_by_year_week[yyw]:
self.activity_by_year_week_peak = self.activity_by_year_week[yyw]
# author stats
if author not in self.authors:
self.authors[author] = {}
# commits
if "last_commit_stamp" not in self.authors[author]:
self.authors[author]["last_commit_stamp"] = stamp
self.authors[author]["first_commit_stamp"] = stamp
self.authors[author]["commits"] = self.authors[author].get("commits", 0) + 1
# author of the month/year
yymm = date.strftime("%Y-%m")
if yymm in self.author_of_month:
self.author_of_month[yymm][author] = self.author_of_month[yymm].get(author, 0) + 1
else:
self.author_of_month[yymm] = {}
self.author_of_month[yymm][author] = 1
self.commits_by_month[yymm] = self.commits_by_month.get(yymm, 0) + 1
yy = date.year
if yy in self.author_of_year:
self.author_of_year[yy][author] = self.author_of_year[yy].get(author, 0) + 1
else:
self.author_of_year[yy] = {}
self.author_of_year[yy][author] = 1
self.commits_by_year[yy] = self.commits_by_year.get(yy, 0) + 1
# authors: active days
yymmdd = date.strftime("%Y-%m-%d")
if "last_active_day" not in self.authors[author]:
self.authors[author]["last_active_day"] = yymmdd
self.authors[author]["active_days"] = 1
elif yymmdd != self.authors[author]["last_active_day"]:
self.authors[author]["last_active_day"] = yymmdd
self.authors[author]["active_days"] += 1
# project: active days
if yymmdd != self.last_active_day:
self.last_active_day = yymmdd
self.active_days.add(yymmdd)
# timezone
self.commits_by_timezone[timezone] = self.commits_by_timezone.get(timezone, 0) + 1
# TODO Optimize this, it's the worst bottleneck
# outputs "<stamp> <files>" for each revision
self.files_by_stamp = {} # stamp -> files
revlines = getpipeoutput(['git rev-list --pretty=format:"%at %T" HEAD', "grep -v ^commit"]).strip().split("\n")
lines = []
for revline in revlines:
time, rev = revline.split(" ")
linecount = self.getFilesInCommit(rev)
lines.append("%d %d" % (int(time), linecount))
self.total_commits = len(lines)
for line in lines:
parts = line.split(" ")
if len(parts) != 2:
continue
(stamp, files) = parts[0:2]
try:
self.files_by_stamp[int(stamp)] = int(files)
except ValueError:
print 'Warning: failed to parse line "%s"' % line
# extensions
self.extensions = {} # extension -> files, lines
lines = getpipeoutput(["git ls-tree -r -z HEAD"]).split("\000")
self.total_files = len(lines)
for line in lines:
if len(line) == 0:
continue
parts = re.split("\s+", line, 4)
sha1 = parts[2]
filename = parts[3]
if filename.find(".") == -1 or filename.rfind(".") == 0:
ext = ""
else:
ext = filename[(filename.rfind(".") + 1) :]
if len(ext) > self.conf["max_ext_length"]:
ext = ""
if ext not in self.extensions:
self.extensions[ext] = {"files": 0, "lines": 0}
self.extensions[ext]["files"] += 1
try:
self.extensions[ext]["lines"] += self.getLinesInBlob(sha1)
except:
print 'Warning: Could not count lines for file "%s"' % line
# line statistics
# outputs:
# N files changed, N insertions (+), N deletions(-)
# <stamp> <author>
self.changes_by_date = {} # stamp -> { files, ins, del }
lines = getpipeoutput(['git log --shortstat --pretty=format:"%at %an"']).split("\n")
lines.reverse()
files = 0
inserted = 0
deleted = 0
total_lines = 0
author = None
for line in lines:
if len(line) == 0:
continue
# <stamp> <author>
if line.find("files changed,") == -1:
pos = line.find(" ")
if pos != -1:
try:
(stamp, author) = (int(line[:pos]), line[pos + 1 :])
self.changes_by_date[stamp] = {
"files": files,
"ins": inserted,
"del": deleted,
"lines": total_lines,
}
if author not in self.authors:
self.authors[author] = {"lines_added": 0, "lines_removed": 0}
self.authors[author]["lines_added"] = self.authors[author].get("lines_added", 0) + inserted
self.authors[author]["lines_removed"] = self.authors[author].get("lines_removed", 0) + deleted
except ValueError:
print 'Warning: unexpected line "%s"' % line
else:
print 'Warning: unexpected line "%s"' % line
else:
numbers = re.findall("\d+", line)
if len(numbers) == 3:
(files, inserted, deleted) = map(lambda el: int(el), numbers)
total_lines += inserted
total_lines -= deleted
self.total_lines_added += inserted
self.total_lines_removed += deleted
else:
print 'Warning: failed to handle line "%s"' % line
(files, inserted, deleted) = (0, 0, 0)
# self.changes_by_date[stamp] = { 'files': files, 'ins': inserted, 'del': deleted }
self.total_lines = total_lines
new_SD_list, new_not_SD = loop_through_ids(
self.ids_social_distance)
assert len(new_SD_list) == 0
# Switch people
for id in new_SD:
self.ids_not_social_distance.remove(id)
self.ids_social_distance.add(id)
for id in new_not_SD:
self.ids_social_distance.remove(id)
self.ids_not_social_distance.add(id)
if render:
pygame.display.flip()
screen.fill((0, 0, 0))
# Frames per second
if self.render_C['fps']: clock.tick(self.render_C['fps'])
self.data_collect.reset(t + 1, last=True)
if __name__ == '__main__':
constants = json.load(open('constants.json'))
# Can save a run as an experiment which saves the data, visualizations and constants in a experiments directory
data_collect = DataCollector(constants,
save_experiment=True,
print_visualizations=True)
# Can print data (look at `data_options` at top of `data_collector.py` for options) and how often to print
data_collect.set_print_options(basic_to_print=['S', 'I', 'R', 'death'],
frequency=1)
CA = CellularAutomation(constants, data_collect)
# Can render each timestep with pygame
CA.run(render=True)
class Target:
## @var config
# A configurator::Configurator object
## @var connection
# A connector::Connector instance used to send commands to XML-RPC connection
## @var is_slave
# Shows if this is a slave target
## @var parameters
# The target parameters
## @var datastream_queue
# The queue where DataCollector thread puts data and observers get it
## @var datastream_poller
# The DataPoller thread
## @var datastream_collector
# The DataCollector thread
## @var signal_structure
# The structure of the signal obtained with getSignalStructure
## @var is_connected
# Checks if the Target is connected or not. Actually the status is simply set bu connect() and disconnect()
## Target initializer.
def __init__(self, slave=False, protocol=None, address=None, port=None, target_name=None):
self.config = Configurator()
self.is_connected = False
self.is_slave = slave
self.parameters = None
self.datastream_queue = None
self.datastream_poller = None
self.datastream_collector = None
self.signal_structure = None
# Sets basic configurations
if protocol:
self.config.setAttr("protocol", protocol)
if address:
self.config.setAttr("address", address)
if port:
self.config.setAttr("port", port)
if target_name:
self.config.setAttr("target_name", target_name)
# Initializes connection
self.connection = Connector(self.config)
## Prints a summary of the target and its status
# TODO: Decidere che informazioni restituire
def getInfo(self):
print "Target name: %s" % self.config.getAttr("target_name")
print "Connection status: %s" % ("Connected" if self.is_connected else "Disconnected")
print "Connection request: %s://%s:%s" % (
self.config.getAttr("protocol"),
self.config.getAttr("address"),
self.config.getAttr("port"),
)
## Setups the queue and the threads needed to read the data stream
# @param sample_time The sample time of the signal
# @param decimation The decimation to use
# @start_threads The flag given by startData() to let the data threads start (False if this method is called by addObserver())
# @return The boolean result of the operation
def setup_data_threads(self, sample_time=None, decimation=None, start_threads=False):
if self.is_slave:
if not self.datastream_queue:
self.datastream_queue = []
# TODO: creare un DataPoller per ogni segnale (non per tutta la traccia) e registrare gli observer sugli specifici DataPoller.
if not self.datastream_poller:
self.datastream_poller = DataPoller(self.datastream_queue, decimation)
# TODO: Il DataCollector deve leggere il numero di segnale e usare la coda specifica per inserirci i dati
if not self.datastream_collector:
# TODO: Fermare i thread alla chiusura del target
self.datastream_collector = DataCollector(
self.config.getAttr("address"), self.config.getAttr("slave_data_port"), self.datastream_queue
)
if start_threads:
print ("Starting data threads...")
if sample_time:
self.datastream_poller.updateSampleTime(sample_time)
self.datastream_poller.start()
self.datastream_collector.start()
print ("Data threads started...")
else:
print ("Can't start data threads without a sample time")
return False
return True
else:
print ("Can't setup the data stream threads on a not-slave server")
return False
## Stops the data threads
# @return The boolean result of the operation
def stop_data_threads(self):
if self.is_slave:
if self.datastream_collector:
print ("Shutting down DataCollector thread")
self.datastream_collector.stop()
self.datastream_collector.join(15.0)
if self.datastream_collector.is_alive():
print ("Can't stop the DataCollector thread, timeout occurs")
else:
print ("DataCollector thread was shut down")
if self.datastream_poller:
print ("Shutting down DataPoller thread")
self.datastream_poller.stop()
self.datastream_poller.join(15.0)
if self.datastream_poller.is_alive():
print ("Can't stop the DataPoller thread, timeout occurs")
else:
print ("DataPoller thread was shut down")
# Clears vars
self.datastream_poller = None
self.datastream_collector = None
self.datastream_queue = None
return True
else:
print ("Can't stop the data stream threads on a not-slave server")
return False
## Add a new observer "update" method to the list of observers.
# @param update_method Is the method of the observer which will be called when new data arrives. The method name is arbitrary, but it must accept a string parameter
# @param signal_number The optional channel to register the observer to
# @param decimation The decimation to use
# @return The ID of the observer, to be used to remove the observer
def addObserver(self, update_method, decimation=None, signal_number=None):
self.setup_data_threads(None, decimation)
return self.datastream_poller.addObserver(update_method, signal_number)
## Removes the observer
# @param observer_id The ID of the observer to remove
# @return False if the observer isn't in the observers list, True if it was correctly removed
def removeObserver(self, observer_id):
return self.datastream_poller.removeObserver(observer_id)
## Connect to the target server. Checks if master or server and call the right method on connector::Connector
# @param password The (optional) password to catch a running master connection
# @return The boolean result of the operation
def connect(self, password=None):
if self.is_slave:
if (
self.connection.connectSlave()
): # TODO: Cambiare il metodo di connessione non passando tutto il Configurator ma usando i dati passati nel costruttore di Connector
self.is_connected = True
self.signal_structure = self.getSignalStructure()
return True
else:
return False
else:
if password:
result = self.connection.Connection_Request(
password
) # Passing self.config to let the Connector set some connection data (e.g. the session_id)
else:
result = self.connection.Connection_Request()
self.is_connected = result
return result
## Checks if master and if affirmative, returns the slave
# @param target_name The target to connect to. If None gets the name from the default configuration
# @return The slave Target or False if not a master target
def getSlave(self, target_name=None):
if self.is_slave:
print "Can't ask for a slave to a slave. Try asking to a master."
return False
else:
# If target is not specified, caller are requesting a "standard" connection to the primary slave target
if not target_name:
target_name = self.config.getAttr("target_name")
slave = Target(
True,
self.config.getAttr("protocol"),
self.config.getAttr("address"),
self.config.getAttr("slave_port"),
target_name,
)
# Sets the session ID
slave.config.setAttr("session_id", self.config.session_id)
return slave
## Disconnects from the XML-RPC server
# @return The boolean result of the operation
def disconnect(self):
if self.is_slave:
result = self.connection.disconnect()
self.is_connected = result
return result
else:
print ("Can't call disconnect on master server")
return False
## Stops target server. This method, called on a master server, calls connector::Connector::stopServer and called on slave, calls connector::Connection::stopSlave
# @return The boolean result of the operation
def stop(self):
if self.is_slave:
# TODO: Stop the data thread if active
result = self.connection.stopSlaveServer()
self.is_connected = result
# Stops data threads
self.stop_data_threads()
return result
else:
result = self.connection.stopMasterServer()
self.is_connected = result
return result
## Closes the session
# @return The boolean result of the operation
def close(self):
result = self.connection.closeSession()
# Stops data threads
self.stop_data_threads()
self.is_connected = result
return result
## Stops the target
# This method closes the data threads (if they exist) then closes the session
# @return The boolean result of the operation
def halt(self):
self.stop_data_threads()
result = self.connection.stopTarget()
self.is_connected = result
return result
## Starts the connected target
# @return The boolean result of the operation
def start(self):
return self.connection.start()
## Starts data transfer for the specified signal
# @param signal_number The number of the signal to start
# @param sample_time The sample time of the signal
# @param decimation The decimation to use
# @return The boolean result of the operation
def startData(self, signal_number, decimation=None):
sample_time = self.getSampleTimeFromSignalNumber(signal_number)
self.setup_data_threads(sample_time, decimation, True)
return self.connection.startData(signal_number, decimation)
## Returns sample time given the signal number
# @param signal_number The number of the signal to analyze
# @return The sample time of the given signal number
def getSampleTimeFromSignalNumber(self, signal_number):
try:
sample_time = None
for signal in self.signal_structure:
if signal["number"] == signal_number:
sample_time = signal["sample_time"]
break
return sample_time
except:
return None
## Stops (all) data transfers
# @param signal_number The number of signal to catch
# @param all_data If True calls StopAllData(). Calls StopData() if False.
# @return The boolean result of the operation
def stopData(self, signal_number=None, all_data=False):
# TODO: Devo chiudere anche i threads dei poller?
if all_data:
return self.connection.stopAllData()
elif signal_number:
return self.connection.stopData(signal_number)
else:
print ("Can't stop data without signal number or all_data flag")
return False
## Get the structure of the signals
# @return The structure of the signals or False if the target has no signals
def getSignalStructure(self):
return self.connection.getSignalStructure()
## Sends new parameters to the server
# @param params The array of parameters to set
# @return The boolean result of the operation
def setParameters(self, params):
return self.connection.setParameters(params)
## Get the parameters from the target server
# @return An array of params. Every param is an hash
def getParameters(self):
self.parameters = self.connection.getParameters()
return self.parameters
## Interface to set a single param
# @param identifier The identifier string to get the param to update
# @param value The value to set to the param
def setParam(self, identifier, value):
if not self.parameters:
self.parameters = self.connection.getParameters()
ids = identifier.split("/")
param_name = ids.pop()
block_name = "/".join(ids)
new_params = []
for param in self.parameters:
if param["block_name"] == block_name and param["param_name"] == param_name:
param["values"] = value
new_params.append(param)
self.parameters = new_params
return self.setParameters(self.parameters)
def run(self):
"""Main loop
"""
self.startCapture()
data_collector = DataCollector(self.dataset)
modeFullFace = 0
modeOneEye = 1
modePictures = 2
modeTwoEyes = 3
modeImgDB = 4
modeDemo = 5
mode = modeDemo
keepLoop = True
current_t = time.clock()
previous_t = current_t
while keepLoop:
pressed_key = cv2.waitKey(1)
current_t = time.clock()
#print('\nclock : ', current_t - previous_t)
previous_t = current_t
img = self.getCameraImage()
if(mode == modeOneEye):
ex = 300
ey = 50
eh = 200
ew = 200
face = Face(img.frame, img.canvas, 0, 0, 640, 480)
eye = Eye(face.frame, face.canvas, ex, ey, ew, eh)
eye.draw(face)
eye.iris.normalizeIris()
elif(mode == modeTwoEyes):
face = Face(img.frame, img.canvas, 0, 0, 640, 480)
left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT)
left_eye.draw(face)
left_eye.iris.normalizeIris()
right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT)
right_eye.draw(face)
right_eye.iris.normalizeIris()
elif(mode == modeFullFace):
face, left_eye, right_eye = img.detectEyes(self.bufferFace, self.bufferLeftEye, self.bufferRightEye)
if face:
face.draw(img)
if left_eye:
left_eye.draw(face)
left_eye.iris.normalizeIris()
if right_eye:
right_eye.draw(face)
right_eye.iris.normalizeIris()
elif(mode == modeImgDB):
img_db = ImageDB("./IR_Database/MMU Iris Database/")
print(img_db.estimateUser(img_db.bits[0]))
exit()
elif(mode == modeDemo):
path = "./TB_Database/"
face = Face(img.frame, img.canvas, 0, 0, 640, 480)
left_eye = Eye(face.frame, face.canvas, 50, 50, 200, 200, EyeType.LEFT)
left_eye.draw(face)
left_eye.iris.normalizeIris()
right_eye = Eye(face.frame, face.canvas, 400, 50, 200, 200, EyeType.RIGHT)
right_eye.draw(face)
right_eye.iris.normalizeIris()
if(ord('0') <= pressed_key & 0xFF <= ord('9')):
print('0-9')
id_person = chr(pressed_key & 0xFF)
while((pressed_key & 0xFF) not in [ord('a'), ord('p')]):
pressed_key = cv2.waitKey(50)
print(pressed_key & 0xFF)
if(pressed_key & 0xFF == ord('a')):
cv2.imwrite(path + id_person + '/left/' + str(int(time.time() * 1000)) + '.bmp', left_eye.frame)
cv2.imwrite(path + id_person + '/right/' + str(int(time.time() * 1000)) + '.bmp', right_eye.frame)
else:
path = "./IR_Database/MMU Iris Database/"
for dir in os.listdir(path):
subpath = path + dir + '/'
if(os.path.isdir(subpath)):
print(dir)
for subdir in os.listdir(subpath):
subsubpath = subpath + subdir + '/'
if(os.path.isdir(subsubpath)):
print('\t', subdir)
for fname in os.listdir(subsubpath):
fpath = subsubpath + fname
if(os.path.isfile(fpath) and os.path.splitext(fname)[1] == '.bmp'):
print('\t\t', fname)
img = Image(cv2.imread(fpath))
face = Face(img.frame, img.canvas)
eye = Eye(face.frame, face.canvas, padding=10)
eye.draw(face)
eye.iris.normalizeIris()
img.show()
pressed_key = cv2.waitKey(1000)
exit()
# Controls
if pressed_key & 0xFF == ord('q'):
keepLoop = False
#if pressed_key & 0xFF == ord('s'):
# self.dataset.save()
#if pressed_key & 0xFF == ord('l'):
# self.dataset.load()
#if pressed_key & 0xFF == ord('m'):
# self.showMoments = not self.showMoments
#if pressed_key & 0xFF == ord('e'):
# self.showEvaluation = not self.showEvaluation
#data_collector.step(img.canvas, pressed_key, left_eye, right_eye)
#txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset))
#cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2)
#cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2)
#if left_eye and right_eye:
# direction = self.dataset.estimateDirection(left_eye.computeMomentVectors(), right_eye.computeMomentVectors())
# txt = 'Estimated direction: {}'.format(direction.name)
# cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2)
# cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2)
img.show()
#if self.showEvaluation:
# fig = self.dataset.showValidationScoreEvolution()
# plt.show()
# self.showEvaluation = False
#if self.showMoments:
# fig = self.dataset.drawVectorizedMoments()
# plt.show()
# # cv2.imshow('moments', self.fig2cv(fig))
# # plt.close(fig)
# self.showMoments = False
self.stopCapture()
def collect_data_and_upload(rc=0):
"""
All the heavy lifting done here
Run through "targets" - could be just one (host, default) or many (containers+host)
"""
# initialize collection targets
# for now we do either containers OR host -- not both at same time
if InsightsClient.options.container_mode:
targets = get_targets()
targets = targets + constants.default_target
else:
targets = constants.default_target
if InsightsClient.options.offline:
logger.warning("Assuming remote branch and leaf value of -1")
pconn = None
branch_info = constants.default_branch_info
else:
pconn = InsightsConnection()
# TODO: change these err msgs to be more meaningful , i.e.
# "could not determine login information"
if pconn:
try:
branch_info = pconn.branch_info()
except requests.ConnectionError:
branch_info = handle_branch_info_error(
"Could not connect to determine branch information")
except LookupError:
branch_info = handle_branch_info_error(
"Could not determine branch information")
pc = InsightsConfig(pconn)
if InsightsClient.options.just_upload:
if not os.path.exists(InsightsClient.options.just_upload):
logger.error('No file %s', InsightsClient.options.just_upload)
return 1
tar_file = InsightsClient.options.just_upload
rc = _do_upload(pconn, tar_file, 'dummy', 0)
return rc
# load config from stdin/file if specified
try:
stdin_config = {}
if InsightsClient.options.from_file:
with open(InsightsClient.options.from_file, 'r') as f:
stdin_config = json.load(f)
elif InsightsClient.options.from_stdin:
stdin_config = json.load(sys.stdin)
if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and
('uploader.json' not in stdin_config or
'sig' not in stdin_config)):
raise ValueError
except:
logger.error('ERROR: Invalid config for %s! Exiting...',
('--from-file' if InsightsClient.options.from_file else '--from-stdin'))
sys.exit(1)
start = time.clock()
collection_rules, rm_conf = pc.get_conf(InsightsClient.options.update, stdin_config)
collection_elapsed = (time.clock() - start)
logger.debug("Rules configuration loaded. Elapsed time: %s", collection_elapsed)
individual_archives = []
for t in targets:
# defaults
archive = None
container_connection = None
mp = None
obfuscate = None
# archive metadata
archive_meta = {}
try:
if t['type'] == 'docker_image':
container_connection = open_image(t['name'])
logging_name = 'Docker image ' + t['name']
archive_meta['display_name'] = get_repotag(t['name'])
archive_meta['docker_id'] = t['name']
if container_connection:
mp = container_connection.get_fs()
else:
logger.error('Could not open %s for analysis', logging_name)
continue
elif t['type'] == 'docker_container':
container_connection = open_container(t['name'])
logging_name = 'Docker container ' + t['name']
archive_meta['display_name'] = t['name']
if container_connection:
mp = container_connection.get_fs()
else:
logger.error('Could not open %s for analysis', logging_name)
continue
elif t['type'] == 'host':
logging_name = determine_hostname()
archive_meta['display_name'] = determine_hostname(InsightsClient.options.display_name)
else:
logger.error('Unexpected analysis target: %s', t['type'])
continue
archive_meta['type'] = t['type'].replace('docker_', '')
archive_meta['product'] = 'Docker'
archive_meta['system_id'] = generate_analysis_target_id(t['type'], t['name'])
collection_start = time.clock()
archive = InsightsArchive(compressor=InsightsClient.options.compressor if not InsightsClient.options.container_mode else "none",
target_name=t['name'])
atexit.register(_delete_archive, archive)
dc = DataCollector(archive,
mountpoint=mp,
target_name=t['name'],
target_type=t['type'])
logger.info('Starting to collect Insights data for %s', logging_name)
dc.run_collection(collection_rules, rm_conf, branch_info)
elapsed = (time.clock() - start)
logger.debug("Data collection complete. Elapsed time: %s", elapsed)
obfuscate = InsightsClient.config.getboolean(APP_NAME, "obfuscate")
# include rule refresh time in the duration
collection_duration = (time.clock() - collection_start) + collection_elapsed
if InsightsClient.options.no_tar_file:
logger.info('See Insights data in %s', dc.archive.archive_dir)
return rc
tar_file = dc.done(collection_rules, rm_conf)
# add archives to list of individual uploads
archive_meta['tar_file'] = tar_file
individual_archives.append(archive_meta)
finally:
# called on loop iter end or unexpected exit
if container_connection:
container_connection.close()
# if multiple targets (container mode), add all archives to single archive
if InsightsClient.options.container_mode:
full_archive = InsightsArchive(compressor=InsightsClient.options.compressor)
for a in individual_archives:
shutil.copy(a['tar_file'], full_archive.archive_dir)
# don't want insights_commands in meta archive
shutil.rmtree(full_archive.cmd_dir)
metadata = _create_metadata_json(individual_archives)
full_archive.add_metadata_to_archive(json.dumps(metadata), 'metadata.json')
full_tar_file = full_archive.create_tar_file(full_archive=True)
# if only one target (regular mode), just upload one
else:
full_archive = archive
full_tar_file = tar_file
if InsightsClient.options.offline or InsightsClient.options.no_upload:
handle_file_output(full_tar_file, full_archive)
return rc
# do the upload
rc = _do_upload(pconn, full_tar_file, logging_name, collection_duration)
if InsightsClient.options.keep_archive:
logger.info('Insights data retained in %s', full_tar_file)
return rc
if obfuscate:
logger.info('Obfuscated Insights data retained in %s',
os.path.dirname(full_tar_file))
full_archive.delete_archive_dir()
return rc
def __init__(self):
self.data_collector = DataCollector()
self.new_location = DataCollector()
class Monitor:
def __init__(self):
self.data_collector = DataCollector()
self.new_location = DataCollector()
def collect_default_data(self):
self.data_collector.filter_radius()
def relocate(self, location=None, coordinates=None, radius: int = 250):
if location is not None:
geolocator = Nominatim(user_agent="team_1_earthquake_detector")
location_latitude = geolocator.geocode(location).latitude
location_longitude = geolocator.geocode(location).longitude
location_coordinates = (location_latitude, location_longitude)
return self.new_location.filter_radius(location_coordinates,
radius)
if coordinates is not None:
return self.new_location.filter_radius(coordinates, radius)
def build_map(self, location=None, coordinates=None, radius=None):
self.map = Map()
if location is None and coordinates is None:
self.map.set_up_map()
self.collect_default_data()
earthquake_overlay = EarthquakeOverlay(
self.data_collector.earthquake_data_clean)
earthquake_overlay.apply_circle_markers(self.map.map)
earthquake_overlay.apply_magnitude_markers(self.map.map)
earthquake_overlay.apply_connective_lines(self.map.map)
earthquake_overlay.apply_heatmap(self.map.map)
tectonic_overlay = TectonicOverlay()
tectonic_overlay.apply_overlay(self.map.map)
tectonic_overlay.add_to_layer_control(self.map.map)
return self.map
if location is not None or coordinates is not None:
self.map.set_up_map(location=coordinates)
self.relocate(location=location,
coordinates=coordinates,
radius=radius)
earthquake_overlay = EarthquakeOverlay(
self.new_location.earthquake_data_clean)
earthquake_overlay.apply_circle_markers(self.map.map)
earthquake_overlay.apply_magnitude_markers(self.map.map)
earthquake_overlay.apply_connective_lines(self.map.map,
coordinates)
earthquake_overlay.apply_heatmap(self.map.map)
tectonic_overlay = TectonicOverlay()
tectonic_overlay.apply_overlay(self.map.map)
tectonic_overlay.add_to_layer_control(self.map.map)
return self.map
def perform_earthquake_analytics(self, location=None, radius=None):
self.data_collector.prep_data()
self.data_collector.filter_radius()
earthquake_analytics = EarthquakeAnalytics(
self.data_collector.earthquake_data,
self.data_collector.earthquake_data_clean)
total_filtered = earthquake_analytics.get_total_filtered_earthquakes(
location=location, radius=radius)
minor_filtered = earthquake_analytics.get_filtered_minor_earthquakes(
location=location, radius=radius)
moderate_filtered = earthquake_analytics.get_filtered_moderate_earthquakes(
location=location, radius=radius)
strong_filtered = earthquake_analytics.get_filtered_strong_earthquakes(
location=location, radius=radius)
closest_filtered = earthquake_analytics.get_closest_filtered_earthquake(
location=location)
place_of_closest_filtered = earthquake_analytics.get_place_of_closest_filtered_earthquake(
location=location)
strongest_filtered = earthquake_analytics.get_strongest_filtered_earthquake(
location=location, radius=radius)
total_worldwide = earthquake_analytics.get_total_earthquakes_worldwide(
)
minor_worldwide = earthquake_analytics.get_minor_earthquakes_worldwide(
)
moderate_worldwide = earthquake_analytics.get_moderate_earthquakes_worldwide(
)
strong_worldwide = earthquake_analytics.get_strong_earthquakes_worldwide(
)
strongest_worldwide = earthquake_analytics.get_strongest_earthquake_worldwide(
)
place_of_strongest_worldwide = earthquake_analytics.get_place_of_strongest_earthquake_worldwide(
)
return total_filtered, minor_filtered, moderate_filtered, strong_filtered, closest_filtered, place_of_closest_filtered,\
strongest_filtered, total_worldwide, minor_worldwide, moderate_worldwide, strong_worldwide, strongest_worldwide, \
place_of_strongest_worldwide
def run(self):
"""Main loop
"""
self.startCapture()
data_collector = DataCollector(self.dataset)
keepLoop = True
while keepLoop:
pressed_key = cv2.waitKey(1)
img = self.getCameraImage()
face, left_eye, right_eye = img.detectEyes(self.bufferFace,
self.bufferLeftEye,
self.bufferRightEye)
if face:
face.draw(img)
if left_eye:
left_eye.draw(face)
if right_eye:
right_eye.draw(face)
# Controls
if pressed_key & 0xFF == ord('q'):
keepLoop = False
if pressed_key & 0xFF == ord('s'):
self.dataset.save()
if pressed_key & 0xFF == ord('l'):
self.dataset.load()
if pressed_key & 0xFF == ord('m'):
self.showMoments = not self.showMoments
if pressed_key & 0xFF == ord('e'):
self.showEvaluation = not self.showEvaluation
data_collector.step(img.canvas, pressed_key, left_eye, right_eye)
txt = 'Dataset: {} (s)ave - (l)oad'.format(len(self.dataset))
cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 29),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2)
cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2)
if left_eye and right_eye:
direction = self.dataset.estimateDirection(
left_eye.computeMomentVectors(),
right_eye.computeMomentVectors())
txt = 'Estimated direction: {}'.format(direction.name)
cv2.putText(img.canvas, txt, (21, img.canvas.shape[0] - 49),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (32, 32, 32), 2)
cv2.putText(img.canvas, txt, (20, img.canvas.shape[0] - 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 126, 255), 2)
img.show()
if self.showEvaluation:
fig = self.dataset.showValidationScoreEvolution()
plt.show()
self.showEvaluation = False
if self.showMoments:
fig = self.dataset.drawVectorizedMoments()
plt.show()
# cv2.imshow('moments', self.fig2cv(fig))
# plt.close(fig)
self.showMoments = False
self.stopCapture()
def get_node_stats(node_name):
node = NodeRepository.by_name(node_name)
data = DataCollector.get_node_info(node)
StatsRepository.save_for_node(node, data)
parser.add_option("-k", action="store", dest="k", type="long", default=3)
parser.add_option("--temp",
action="store",
dest="temp_rate",
type="float",
default=0.9)
parser.add_option("--iters",
action="store",
dest="iters",
type="float",
default=100)
parser.add_option("--max_t",
action="store",
dest="max_temp",
type="float",
default=100)
parser.add_option("--min_t",
action="store",
dest="min_temp",
type="float",
default=0.5)
parser.add_option("--hh", action="store", type="string", dest="hh")
global options
(options, _) = parser.parse_args()
# data collector configuration
global collector
collector = DataCollector()
def __init__(self, conf):
DataCollector.__init__(self)
self.conf = conf
